[TSan] Fix madvise(MADV_NOHUGEPAGE) for meta shadow memory

Summary:
Move madvise(MADV_NOHUGEPAGE) for the meta shadow memory after the meta
shadow memory is mapped (currently it silently fails with ENOMEM).

Add a diagnostic message to detect similar problems in the future.

Reviewers: dvyukov

Subscribers: kubamracek, delcypher, #sanitizers, llvm-commits

Differential Revision: https://reviews.llvm.org/D48097

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@334624 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 25 insertions, 9 deletions

diff --git a/lib/tsan/rtl/tsan_platform_posix.cc b/lib/tsan/rtl/tsan_platform_posix.cc
index e4f90a811..22b649ef3 100644
--- a/lib/tsan/rtl/tsan_platform_posix.cc
+++ b/lib/tsan/rtl/tsan_platform_posix.cc
@@ -23,6 +23,27 @@
 
 namespace __tsan {
 
+static const char kShadowMemoryMappingWarning[] =
+    "FATAL: %s can not madvise shadow region [%zx, %zx] with %s\n";
+
+static void NoHugePagesInShadow(uptr addr, uptr size) {
+  if (common_flags()->no_huge_pages_for_shadow)
+    if (!NoHugePagesInRegion(addr, size)) {
+      Printf(kShadowMemoryMappingWarning, SanitizerToolName, addr, addr + size,
+             "MADV_NOHUGEPAGE");
+      Die();
+    }
+}
+
+static void DontDumpShadow(uptr addr, uptr size) {
+  if (common_flags()->use_madv_dontdump)
+    if (!DontDumpShadowMemory(addr, size)) {
+      Printf(kShadowMemoryMappingWarning, SanitizerToolName, addr, addr + size,
+             "MADV_DONTDUMP");
+      Die();
+    }
+}
+
 #if !SANITIZER_GO
 void InitializeShadowMemory() {
   // Map memory shadow.
@@ -74,14 +95,9 @@ void InitializeShadowMemory() {
     DCHECK(0);
   }
 #endif
-  NoHugePagesInRegion(MemToShadow(kMadviseRangeBeg),
+  NoHugePagesInShadow(MemToShadow(kMadviseRangeBeg),
                       kMadviseRangeSize * kShadowMultiplier);
-  // Meta shadow is compressing and we don't flush it,
-  // so it makes sense to mark it as NOHUGEPAGE to not over-allocate memory.
-  // On one program it reduces memory consumption from 5GB to 2.5GB.
-  NoHugePagesInRegion(MetaShadowBeg(), MetaShadowEnd() - MetaShadowBeg());
-  if (common_flags()->use_madv_dontdump)
-    DontDumpShadowMemory(ShadowBeg(), ShadowEnd() - ShadowBeg());
+  DontDumpShadow(ShadowBeg(), ShadowEnd() - ShadowBeg());
   DPrintf("memory shadow: %zx-%zx (%zuGB)\n",
       ShadowBeg(), ShadowEnd(),
       (ShadowEnd() - ShadowBeg()) >> 30);
@@ -96,8 +112,8 @@ void InitializeShadowMemory() {
                "to link with -pie (%p, %p).\n", meta, MetaShadowBeg());
     Die();
   }
-  if (common_flags()->use_madv_dontdump)
-    DontDumpShadowMemory(meta, meta_size);
+  NoHugePagesInShadow(meta, meta_size);
+  DontDumpShadow(meta, meta_size);
   DPrintf("meta shadow: %zx-%zx (%zuGB)\n",
       meta, meta + meta_size, meta_size >> 30);