[compiler-rt] r351620 - [hwasan] Madvise away unused shadow.
Evgeniy Stepanov via llvm-commits
llvm-commits at lists.llvm.org
Fri Jan 18 17:54:10 PST 2019
Author: eugenis
Date: Fri Jan 18 17:54:09 2019
New Revision: 351620
URL: http://llvm.org/viewvc/llvm-project?rev=351620&view=rev
Log:
[hwasan] Madvise away unused shadow.
Summary:
Whenever a large shadow region is tagged to zero, madvise(DONT_NEED)
as much of it as possible.
This reduces shadow RSS on Android by 45% or so, and total memory use
by 2-4%, probably even more on long running multithreaded programs.
CPU time seems to be in the noise.
Reviewers: kcc, pcc
Subscribers: srhines, kubamracek, llvm-commits
Differential Revision: https://reviews.llvm.org/D56757
Added:
compiler-rt/trunk/test/hwasan/TestCases/Linux/release-shadow.c
Modified:
compiler-rt/trunk/lib/hwasan/hwasan.cc
compiler-rt/trunk/lib/hwasan/hwasan_poisoning.cc
compiler-rt/trunk/lib/sanitizer_common/sanitizer_linux.h
Modified: compiler-rt/trunk/lib/hwasan/hwasan.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/hwasan/hwasan.cc?rev=351620&r1=351619&r2=351620&view=diff
==============================================================================
--- compiler-rt/trunk/lib/hwasan/hwasan.cc (original)
+++ compiler-rt/trunk/lib/hwasan/hwasan.cc Fri Jan 18 17:54:09 2019
@@ -88,6 +88,8 @@ static void InitializeFlags() {
cf.check_printf = false;
cf.intercept_tls_get_addr = true;
cf.exitcode = 99;
+ // 8 shadow pages ~512kB, small enough to cover common stack sizes.
+ cf.clear_shadow_mmap_threshold = 4096 * (SANITIZER_ANDROID ? 2 : 8);
// Sigtrap is used in error reporting.
cf.handle_sigtrap = kHandleSignalExclusive;
Modified: compiler-rt/trunk/lib/hwasan/hwasan_poisoning.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/hwasan/hwasan_poisoning.cc?rev=351620&r1=351619&r2=351620&view=diff
==============================================================================
--- compiler-rt/trunk/lib/hwasan/hwasan_poisoning.cc (original)
+++ compiler-rt/trunk/lib/hwasan/hwasan_poisoning.cc Fri Jan 18 17:54:09 2019
@@ -16,6 +16,7 @@
#include "hwasan_mapping.h"
#include "interception/interception.h"
#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_linux.h"
namespace __hwasan {
@@ -24,7 +25,22 @@ uptr TagMemoryAligned(uptr p, uptr size,
CHECK(IsAligned(size, kShadowAlignment));
uptr shadow_start = MemToShadow(p);
uptr shadow_size = MemToShadowSize(size);
- internal_memset((void *)shadow_start, tag, shadow_size);
+
+ uptr page_size = GetPageSizeCached();
+ uptr page_start = RoundUpTo(shadow_start, page_size);
+ uptr page_end = RoundDownTo(shadow_start + shadow_size, page_size);
+ uptr threshold = common_flags()->clear_shadow_mmap_threshold;
+ if (SANITIZER_LINUX &&
+ UNLIKELY(page_end >= page_start + threshold && tag == 0)) {
+ internal_memset((void *)shadow_start, tag, page_start - shadow_start);
+ internal_memset((void *)page_end, tag,
+ shadow_start + shadow_size - page_end);
+ // For an anonymous private mapping MADV_DONTNEED will return a zero page on
+ // Linux.
+ ReleaseMemoryPagesToOSAndZeroFill(page_start, page_end);
+ } else {
+ internal_memset((void *)shadow_start, tag, shadow_size);
+ }
return AddTagToPointer(p, tag);
}
Modified: compiler-rt/trunk/lib/sanitizer_common/sanitizer_linux.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/sanitizer_common/sanitizer_linux.h?rev=351620&r1=351619&r2=351620&view=diff
==============================================================================
--- compiler-rt/trunk/lib/sanitizer_common/sanitizer_linux.h (original)
+++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_linux.h Fri Jan 18 17:54:09 2019
@@ -106,6 +106,17 @@ bool LibraryNameIs(const char *full_name
// Call cb for each region mapped by map.
void ForEachMappedRegion(link_map *map, void (*cb)(const void *, uptr));
+// Releases memory pages entirely within the [beg, end] address range.
+// The pages no longer count toward RSS; reads are guaranteed to return 0.
+// Requires (but does not verify!) that pages are MAP_PRIVATE.
+INLINE void ReleaseMemoryPagesToOSAndZeroFill(uptr beg, uptr end) {
+ // man madvise on Linux promises zero-fill for anonymous private pages.
+ // Testing shows the same behaviour for private (but not anonymous) mappings
+ // of shm_open() files, as long as the underlying file is untouched.
+ CHECK(SANITIZER_LINUX);
+ ReleaseMemoryPagesToOS(beg, end);
+}
+
#if SANITIZER_ANDROID
#if defined(__aarch64__)
Added: compiler-rt/trunk/test/hwasan/TestCases/Linux/release-shadow.c
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/test/hwasan/TestCases/Linux/release-shadow.c?rev=351620&view=auto
==============================================================================
--- compiler-rt/trunk/test/hwasan/TestCases/Linux/release-shadow.c (added)
+++ compiler-rt/trunk/test/hwasan/TestCases/Linux/release-shadow.c Fri Jan 18 17:54:09 2019
@@ -0,0 +1,70 @@
+// Test that tagging a large region to 0 reduces RSS.
+// RUN: %clang_hwasan -mllvm -hwasan-instrument-stack=0 %s -o %t && %run %t 2>&1
+
+#include <assert.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <sanitizer/hwasan_interface.h>
+
+const unsigned char kTag = 42;
+const size_t kNumShadowPages = 256;
+const size_t kNumPages = 16 * kNumShadowPages;
+const size_t kPageSize = 4096;
+const size_t kMapSize = kNumPages * kPageSize;
+
+void sync_rss() {
+ char *page = (char *)mmap(0, kPageSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
+ // Linux kernel updates RSS counters after a set number of page faults.
+ for (int i = 0; i < 1000; ++i) {
+ page[0] = 42;
+ madvise(page, kPageSize, MADV_DONTNEED);
+ }
+ munmap(page, kPageSize);
+}
+
+size_t current_rss() {
+ sync_rss();
+ int statm_fd = open("/proc/self/statm", O_RDONLY);
+ assert(statm_fd >= 0);
+
+ char buf[100];
+ assert(read(statm_fd, &buf, sizeof(buf)) > 0);
+ size_t size, rss;
+ assert(sscanf(buf, "%zu %zu", &size, &rss) == 2);
+
+ close(statm_fd);
+ return rss;
+}
+
+void test_rss_difference(void *p) {
+ __hwasan_tag_memory(p, kTag, kMapSize);
+ size_t rss_before = current_rss();
+ __hwasan_tag_memory(p, 0, kMapSize);
+ size_t rss_after = current_rss();
+ fprintf(stderr, "%zu -> %zu\n", rss_before, rss_after);
+ assert(rss_before > rss_after);
+ size_t diff = rss_before - rss_after;
+ fprintf(stderr, "diff %zu\n", diff);
+ // Check that the difference is at least close to kNumShadowPages.
+ assert(diff > kNumShadowPages / 4 * 3);
+}
+
+int main() {
+ fprintf(stderr, "starting rss %zu\n", current_rss());
+ fprintf(stderr, "shadow pages: %zu\n", kNumShadowPages);
+
+ void *p = mmap(0, kMapSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
+ fprintf(stderr, "p = %p\n", p);
+
+ test_rss_difference(p);
+ test_rss_difference(p);
+ test_rss_difference(p);
+
+ return 0;
+}
More information about the llvm-commits
mailing list