[compiler-rt] 3b3f8c5 - [asan] Re-exec without ASLR if needed on 32-bit Linux (#131975)

[via llvm-commits]

Author: Thurston Dang
Date: 2025-03-20T08:02:48-07:00
New Revision: 3b3f8c50b646600dc1846c7d3d018fdb39fbfe0b

URL: https://github.com/llvm/llvm-project/commit/3b3f8c50b646600dc1846c7d3d018fdb39fbfe0b
DIFF: https://github.com/llvm/llvm-project/commit/3b3f8c50b646600dc1846c7d3d018fdb39fbfe0b.diff

LOG: [asan] Re-exec without ASLR if needed on 32-bit Linux (#131975)

High-entropy ASLR allows up to 16-bits of entropy (2**16 4KB pages ==
256MB; a bit more in practice because of implementation details), which
is a significant chunk of the user address space on 32-bit systems (4GB
or less). This, combined with ASan's shadow (512MB) and ASan's fixed
shadow offset (512MB), makes it possible for large binaries to fail to
map the shadow.

This patch changes ASan to do a one-time re-exec without ASLR if it
cannot map the shadow, thus reclaiming the ~256MB of address space.

Alternatives considered:
1) We don't lower ASan's fixed shadow offset, because that would limit
non-PIE binaries.
2) We don't switch to a dynamic shadow offset, because ASan for 32-bit
Linux relies on the compile-time constant offset to optimize its
instrumentation and compiler-rt.

This is loosely inspired by
https://github.com/llvm/llvm-project/pull/78351,
https://github.com/llvm/llvm-project/pull/85142, and
https://github.com/llvm/llvm-project/pull/85674, though those were
required because there were no static shadow mappings that could fully
shadow the range of user mappings; this is not the case for ASan.

Added: 
    

Modified: 
    compiler-rt/lib/asan/asan_internal.h
    compiler-rt/lib/asan/asan_linux.cpp
    compiler-rt/lib/asan/asan_mac.cpp
    compiler-rt/lib/asan/asan_shadow_setup.cpp
    compiler-rt/lib/asan/asan_win.cpp

Removed: 
    


################################################################################
diff  --git a/compiler-rt/lib/asan/asan_internal.h b/compiler-rt/lib/asan/asan_internal.h
index 06dfc4b177339..8205e19fdab7f 100644
--- a/compiler-rt/lib/asan/asan_internal.h
+++ b/compiler-rt/lib/asan/asan_internal.h
@@ -82,6 +82,7 @@ void ReplaceSystemMalloc();
 uptr FindDynamicShadowStart();
 void AsanCheckDynamicRTPrereqs();
 void AsanCheckIncompatibleRT();
+void TryReExecWithoutASLR();
 
 // Unpoisons platform-specific stacks.
 // Returns true if all stacks have been unpoisoned.

diff  --git a/compiler-rt/lib/asan/asan_linux.cpp b/compiler-rt/lib/asan/asan_linux.cpp
index 4cabca388ca9a..7620ecb1e81f9 100644
--- a/compiler-rt/lib/asan/asan_linux.cpp
+++ b/compiler-rt/lib/asan/asan_linux.cpp
@@ -21,6 +21,7 @@
 #  include <pthread.h>
 #  include <stdio.h>
 #  include <sys/mman.h>
+#  include <sys/personality.h>
 #  include <sys/resource.h>
 #  include <sys/syscall.h>
 #  include <sys/time.h>
@@ -107,6 +108,37 @@ void FlushUnneededASanShadowMemory(uptr p, uptr size) {
   ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
 }
 
+void ReExecWithoutASLR() {
+  // ASLR personality check.
+  // Caution: 'personality' is sometimes forbidden by sandboxes, so only call
+  // this function as a last resort (when the memory mapping is incompatible
+  // and ASan would fail anyway).
+  int old_personality = personality(0xffffffff);
+  if (old_personality == -1) {
+    VReport(1, "WARNING: unable to run personality check.\n");
+    return;
+  }
+
+  bool aslr_on = (old_personality & ADDR_NO_RANDOMIZE) == 0;
+
+  if (aslr_on) {
+    // Disable ASLR if the memory layout was incompatible.
+    // Alternatively, we could just keep re-execing until we get lucky
+    // with a compatible randomized layout, but the risk is that if it's
+    // not an ASLR-related issue, we will be stuck in an infinite loop of
+    // re-execing (unless we change ReExec to pass a parameter of the
+    // number of retries allowed.)
+    VReport(1,
+            "WARNING: AddressSanitizer: memory layout is incompatible, "
+            "possibly due to high-entropy ASLR.\n"
+            "Re-execing with fixed virtual address space.\n"
+            "N.B. reducing ASLR entropy is preferable.\n");
+    CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1);
+
+    ReExec();
+  }
+}
+
 #  if SANITIZER_ANDROID
 // FIXME: should we do anything for Android?
 void AsanCheckDynamicRTPrereqs() {}

diff  --git a/compiler-rt/lib/asan/asan_mac.cpp b/compiler-rt/lib/asan/asan_mac.cpp
index bfc349223258b..be513a03ed5cd 100644
--- a/compiler-rt/lib/asan/asan_mac.cpp
+++ b/compiler-rt/lib/asan/asan_mac.cpp
@@ -55,6 +55,9 @@ uptr FindDynamicShadowStart() {
                           GetMmapGranularity());
 }
 
+// Not used.
+void TryReExecWithoutASLR() {}
+
 // No-op. Mac does not support static linkage anyway.
 void AsanCheckDynamicRTPrereqs() {}
 

diff  --git a/compiler-rt/lib/asan/asan_shadow_setup.cpp b/compiler-rt/lib/asan/asan_shadow_setup.cpp
index fc6de39622b51..e66b8af1d2c30 100644
--- a/compiler-rt/lib/asan/asan_shadow_setup.cpp
+++ b/compiler-rt/lib/asan/asan_shadow_setup.cpp
@@ -109,6 +109,13 @@ void InitializeShadowMemory() {
     ProtectGap(kShadowGap2Beg, kShadowGap2End - kShadowGap2Beg + 1);
     ProtectGap(kShadowGap3Beg, kShadowGap3End - kShadowGap3Beg + 1);
   } else {
+    // The shadow mappings can shadow the entire user address space. However,
+    // on 32-bit systems, the maximum ASLR entropy (currently up to 16-bits
+    // == 256MB) is a significant chunk of the address space; reclaiming it by
+    // disabling ASLR might allow chonky binaries to run.
+    if (sizeof(uptr) == 32)
+      TryReExecWithoutASLR();
+
     Report(
         "Shadow memory range interleaves with an existing memory mapping. "
         "ASan cannot proceed correctly. ABORTING.\n");

diff  --git a/compiler-rt/lib/asan/asan_win.cpp b/compiler-rt/lib/asan/asan_win.cpp
index 027340280e068..845408ac38abc 100644
--- a/compiler-rt/lib/asan/asan_win.cpp
+++ b/compiler-rt/lib/asan/asan_win.cpp
@@ -279,6 +279,9 @@ uptr FindDynamicShadowStart() {
                           GetMmapGranularity());
 }
 
+// Not used
+void TryReExecWithoutASLR() {}
+
 void AsanCheckDynamicRTPrereqs() {}
 
 void AsanCheckIncompatibleRT() {}