[PATCH] D11999: [windows] Fix deadlock on mmap failure due to CHECK recursion

[Reid Kleckner via llvm-commits]

rnk created this revision.
rnk added a reviewer: samsonov.
rnk added a subscriber: llvm-commits.

Printing a stacktrace acquires a spinlock, and the sanitizer spinlocks
aren't re-entrant. Avoid the problem by reusing the logic we already
have on Posix.

This failure mode is already exercised by the existing mmap_limit_mb.cc
test case. It will be enabled in a forthcoming change, so I didn't add
standalone tests for this change.

http://reviews.llvm.org/D11999

Files:
  lib/sanitizer_common/sanitizer_common.cc
  lib/sanitizer_common/sanitizer_common.h
  lib/sanitizer_common/sanitizer_posix.cc
  lib/sanitizer_common/sanitizer_win.cc

Index: lib/sanitizer_common/sanitizer_win.cc
===================================================================
--- lib/sanitizer_common/sanitizer_win.cc
+++ lib/sanitizer_common/sanitizer_win.cc
@@ -85,12 +85,8 @@
 
 void *MmapOrDie(uptr size, const char *mem_type) {
   void *rv = VirtualAlloc(0, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
-  if (rv == 0) {
-    Report("ERROR: %s failed to "
-           "allocate 0x%zx (%zd) bytes of %s (error code: %d)\n",
-           SanitizerToolName, size, size, mem_type, GetLastError());
-    CHECK("unable to mmap" && 0);
-  }
+  if (rv == 0)
+    ReportMmapFailureAndDie(size, mem_type, GetLastError());
   return rv;
 }
 
Index: lib/sanitizer_common/sanitizer_posix.cc
===================================================================
--- lib/sanitizer_common/sanitizer_posix.cc
+++ lib/sanitizer_common/sanitizer_posix.cc
@@ -117,21 +117,8 @@
                             PROT_READ | PROT_WRITE,
                             MAP_PRIVATE | MAP_ANON, -1, 0);
   int reserrno;
-  if (internal_iserror(res, &reserrno)) {
-    static int recursion_count;
-    if (recursion_count) {
-      // The Report() and CHECK calls below may call mmap recursively and fail.
-      // If we went into recursion, just die.
-      RawWrite("ERROR: Failed to mmap\n");
-      Die();
-    }
-    recursion_count++;
-    Report("ERROR: %s failed to "
-           "allocate 0x%zx (%zd) bytes of %s (errno: %d)\n",
-           SanitizerToolName, size, size, mem_type, reserrno);
-    DumpProcessMap();
-    CHECK("unable to mmap" && 0);
-  }
+  if (internal_iserror(res, &reserrno))
+    ReportMmapFailureAndDie(size, mem_type, reserrno);
   IncreaseTotalMmap(size);
   return (void *)res;
 }
Index: lib/sanitizer_common/sanitizer_common.h
===================================================================
--- lib/sanitizer_common/sanitizer_common.h
+++ lib/sanitizer_common/sanitizer_common.h
@@ -306,6 +306,8 @@
 void NORETURN Die();
 void NORETURN
 CheckFailed(const char *file, int line, const char *cond, u64 v1, u64 v2);
+void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type,
+                                      error_t err);
 
 // Set the name of the current thread to 'name', return true on succees.
 // The name may be truncated to a system-dependent limit.
Index: lib/sanitizer_common/sanitizer_common.cc
===================================================================
--- lib/sanitizer_common/sanitizer_common.cc
+++ lib/sanitizer_common/sanitizer_common.cc
@@ -140,6 +140,23 @@
   Die();
 }
 
+void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type,
+                                      error_t err) {
+  static int recursion_count;
+  if (recursion_count) {
+    // The Report() and CHECK calls below may call mmap recursively and fail.
+    // If we went into recursion, just die.
+    RawWrite("ERROR: Failed to mmap\n");
+    Die();
+  }
+  recursion_count++;
+  Report("ERROR: %s failed to "
+         "allocate 0x%zx (%zd) bytes of %s (error code: %d)\n",
+         SanitizerToolName, size, size, mem_type, err);
+  DumpProcessMap();
+  UNREACHABLE("unable to mmap");
+}
+
 bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size,
                       uptr *read_len, uptr max_len, error_t *errno_p) {
   uptr PageSize = GetPageSizeCached();


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D11999.32002.patch
Type: text/x-patch
Size: 3357 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20150812/65c23508/attachment.bin>