[asan] Be more careful and verbose when allocating dynamic shadow memory

FindAvailableMemoryRange can currently overwrite existing memory (by restricting the VM below addresses that are already used). This patch adds a check to make sure we don't restrict the VM space too much. We are also now more explicit about why the lookup failed and print out verbose values.

Differential Revision: https://reviews.llvm.org/D43318



git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@326106 91177308-0d34-0410-b5e6-96231b3b80d8

5 files changed, 36 insertions, 12 deletions

diff --git a/lib/asan/asan_mac.cc b/lib/asan/asan_mac.cc
index b7af1a586..17a0ec577 100644
--- a/lib/asan/asan_mac.cc
+++ b/lib/asan/asan_mac.cc
@@ -62,16 +62,36 @@ uptr FindDynamicShadowStart() {
   uptr space_size = kHighShadowEnd + left_padding;
 
   uptr largest_gap_found = 0;
-  uptr shadow_start = FindAvailableMemoryRange(space_size, alignment,
-                                               granularity, &largest_gap_found);
+  uptr max_occupied_addr = 0;
+  VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size);
+  uptr shadow_start =
+      FindAvailableMemoryRange(space_size, alignment, granularity,
+                               &largest_gap_found, &max_occupied_addr);
   // If the shadow doesn't fit, restrict the address space to make it fit.
   if (shadow_start == 0) {
+    VReport(
+        2,
+        "Shadow doesn't fit, largest_gap_found = %p, max_occupied_addr = %p\n",
+        largest_gap_found, max_occupied_addr);
     uptr new_max_vm = RoundDownTo(largest_gap_found << SHADOW_SCALE, alignment);
+    if (new_max_vm < max_occupied_addr) {
+      Report("Unable to find a memory range for dynamic shadow.\n");
+      Report(
+          "space_size = %p, largest_gap_found = %p, max_occupied_addr = %p, "
+          "new_max_vm = %p\n",
+          space_size, largest_gap_found, max_occupied_addr, new_max_vm);
+      CHECK(0 && "cannot place shadow");
+    }
     RestrictMemoryToMaxAddress(new_max_vm);
     kHighMemEnd = new_max_vm - 1;
     space_size = kHighShadowEnd + left_padding;
-    shadow_start =
-        FindAvailableMemoryRange(space_size, alignment, granularity, nullptr);
+    VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size);
+    shadow_start = FindAvailableMemoryRange(space_size, alignment, granularity,
+                                            nullptr, nullptr);
+    if (shadow_start == 0) {
+      Report("Unable to find a memory range after restricting VM.\n");
+      CHECK(0 && "cannot place shadow after restricting vm");
+    }
   }
   CHECK_NE((uptr)0, shadow_start);
   CHECK(IsAligned(shadow_start, alignment));
diff --git a/lib/sanitizer_common/sanitizer_common.h b/lib/sanitizer_common/sanitizer_common.h
index 1fbaee7e3..d06d74eeb 100644
--- a/lib/sanitizer_common/sanitizer_common.h
+++ b/lib/sanitizer_common/sanitizer_common.h
@@ -112,7 +112,7 @@ bool MprotectReadOnly(uptr addr, uptr size);
 
 // Find an available address space.
 uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
-                              uptr *largest_gap_found);
+                              uptr *largest_gap_found, uptr *max_occupied_addr);
 
 // Used to check if we can map shadow memory to a fixed location.
 bool MemoryRangeIsAvailable(uptr range_start, uptr range_end);
diff --git a/lib/sanitizer_common/sanitizer_linux.cc b/lib/sanitizer_common/sanitizer_linux.cc
index 4cea03f7f..c4352f993 100644
--- a/lib/sanitizer_common/sanitizer_linux.cc
+++ b/lib/sanitizer_common/sanitizer_linux.cc
@@ -1859,7 +1859,8 @@ void CheckNoDeepBind(const char *filename, int flag) {
 }
 
 uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
-                              uptr *largest_gap_found) {
+                              uptr *largest_gap_found,
+                              uptr *max_occupied_addr) {
   UNREACHABLE("FindAvailableMemoryRange is not available");
   return 0;
 }
diff --git a/lib/sanitizer_common/sanitizer_mac.cc b/lib/sanitizer_common/sanitizer_mac.cc
index 8fa61318a..3eb023ebf 100644
--- a/lib/sanitizer_common/sanitizer_mac.cc
+++ b/lib/sanitizer_common/sanitizer_mac.cc
@@ -887,10 +887,9 @@ uptr GetMaxVirtualAddress() {
   return GetMaxUserVirtualAddress();
 }
 
-uptr FindAvailableMemoryRange(uptr shadow_size,
-                              uptr alignment,
-                              uptr left_padding,
-                              uptr *largest_gap_found) {
+uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
+                              uptr *largest_gap_found,
+                              uptr *max_occupied_addr) {
   typedef vm_region_submap_short_info_data_64_t RegionInfo;
   enum { kRegionInfoSize = VM_REGION_SUBMAP_SHORT_INFO_COUNT_64 };
   // Start searching for available memory region past PAGEZERO, which is
@@ -902,6 +901,7 @@ uptr FindAvailableMemoryRange(uptr shadow_size,
   mach_vm_address_t free_begin = start_address;
   kern_return_t kr = KERN_SUCCESS;
   if (largest_gap_found) *largest_gap_found = 0;
+  if (max_occupied_addr) *max_occupied_addr = 0;
   while (kr == KERN_SUCCESS) {
     mach_vm_size_t vmsize = 0;
     natural_t depth = 0;
@@ -913,13 +913,15 @@ uptr FindAvailableMemoryRange(uptr shadow_size,
       // No more regions beyond "address", consider the gap at the end of VM.
       address = GetMaxVirtualAddress() + 1;
       vmsize = 0;
+    } else {
+      if (max_occupied_addr) *max_occupied_addr = address + vmsize;
     }
     if (free_begin != address) {
       // We found a free region [free_begin..address-1].
       uptr gap_start = RoundUpTo((uptr)free_begin + left_padding, alignment);
       uptr gap_end = RoundDownTo((uptr)address, alignment);
       uptr gap_size = gap_end > gap_start ? gap_end - gap_start : 0;
-      if (shadow_size < gap_size) {
+      if (size < gap_size) {
         return gap_start;
       }
 
diff --git a/lib/sanitizer_common/sanitizer_win.cc b/lib/sanitizer_common/sanitizer_win.cc
index 95533eb5b..319ed6430 100644
--- a/lib/sanitizer_common/sanitizer_win.cc
+++ b/lib/sanitizer_common/sanitizer_win.cc
@@ -331,7 +331,8 @@ void DontDumpShadowMemory(uptr addr, uptr length) {
 }
 
 uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
-                              uptr *largest_gap_found) {
+                              uptr *largest_gap_found,
+                              uptr *max_occupied_addr) {
   uptr address = 0;
   while (true) {
     MEMORY_BASIC_INFORMATION info;