[esan] Add handling of large stack size rlimits

Summary:
Adds detection of large stack size rlimits (over 1 TB or unlimited), which
results in an mmap location that our shadow mapping does not support.  We
re-exec the application in this situation.  Adds a test of this behavior.

Adds general detection of mmap regions outside of our app regions.  In the
future we want to try to adaptively handle these but for now we abort.

Moves the existing Linux-specific mmap code into a platform-specific file
where the new rlimit code lives.

Reviewers: eugenis

Subscribers: vitalybuka, zhaoqin, kcc, aizatsky, llvm-commits, kubabrecka

Differential Revision: http://reviews.llvm.org/D20745

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@271079 91177308-0d34-0410-b5e6-96231b3b80d8

6 files changed, 104 insertions, 39 deletions

diff --git a/lib/esan/CMakeLists.txt b/lib/esan/CMakeLists.txt
index 3bdf32b72..24bed26a2 100644
--- a/lib/esan/CMakeLists.txt
+++ b/lib/esan/CMakeLists.txt
@@ -12,6 +12,7 @@ set(ESAN_SOURCES
   esan_flags.cpp
   esan_interface.cpp
   esan_interceptors.cpp
+  esan_linux.cpp
   cache_frag.cpp
   working_set.cpp)
 
diff --git a/lib/esan/esan.cpp b/lib/esan/esan.cpp
index b2110bb04..2711e4b97 100644
--- a/lib/esan/esan.cpp
+++ b/lib/esan/esan.cpp
@@ -124,6 +124,8 @@ static bool verifyShadowScheme() {
 #endif
 
 static void initializeShadow() {
+  verifyAddressSpace();
+
   DCHECK(verifyShadowScheme());
 
   Mapping.initialize(ShadowScale[WhichTool]);
diff --git a/lib/esan/esan.h b/lib/esan/esan.h
index 2dc8d1991..5be2240b3 100644
--- a/lib/esan/esan.h
+++ b/lib/esan/esan.h
@@ -26,6 +26,7 @@
 #ifndef ESAN_H
 #define ESAN_H
 
+#include "interception/interception.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "esan_interface_internal.h"
 
@@ -44,6 +45,11 @@ void processCompilationUnitExit(void *Ptr);
 void processRangeAccess(uptr PC, uptr Addr, int Size, bool IsWrite);
 void initializeInterceptors();
 
+// Platform-dependent routines.
+void verifyAddressSpace();
+bool fixMmapAddr(void **Addr, SIZE_T Size, int Flags);
+uptr checkMmapResult(uptr Addr, SIZE_T Size);
+
 } // namespace __esan
 
 #endif // ESAN_H
diff --git a/lib/esan/esan_interceptors.cpp b/lib/esan/esan_interceptors.cpp
index 8c927c692..a9489d29e 100644
--- a/lib/esan/esan_interceptors.cpp
+++ b/lib/esan/esan_interceptors.cpp
@@ -21,19 +21,6 @@
 
 using namespace __esan; // NOLINT
 
-// FIXME: if this gets more complex as more platforms are added we may
-// want to split pieces into separate platform-specific files.
-#if SANITIZER_LINUX
-// Sanitizer runtimes in general want to avoid including system headers.
-// We define the few constants we need here:
-const int EINVAL = 22; // from /usr/include/asm-generic/errno-base.h
-const int MAP_FIXED = 0x10; // from /usr/include/sys/mman.h
-extern "C" int *__errno_location();
-#define errno (*__errno_location())
-#else
-#error Other platforms are not yet supported.
-#endif
-
 #define CUR_PC() (StackTrace::GetCurrentPc())
 
 //===----------------------------------------------------------------------===//
@@ -343,35 +330,12 @@ INTERCEPTOR(int, rmdir, char *path) {
 // These are candidates for sharing with all sanitizers if shadow memory
 // support is also standardized.
 
-static bool fixMmapAddr(void **addr, SIZE_T sz, int flags) {
-  if (*addr) {
-    uptr AppStart, AppEnd;
-    bool SingleApp = false;
-    for (int i = 0; getAppRegion(i, &AppStart, &AppEnd); ++i) {
-      if ((uptr)*addr >= AppStart && (uptr)*addr + sz - 1 <= AppEnd) {
-        SingleApp = true;
-        break;
-      }
-    }
-    if (!SingleApp) {
-      VPrintf(1, "mmap conflict: [%p-%p) is not in an app region\n",
-              *addr, (uptr)*addr + sz);
-      if (flags & MAP_FIXED) {
-        errno = EINVAL;
-        return false;
-      } else {
-        *addr = 0;
-      }
-    }
-  }
-  return true;
-}
-
 INTERCEPTOR(void *, mmap, void *addr, SIZE_T sz, int prot, int flags,
                  int fd, OFF_T off) {
   if (!fixMmapAddr(&addr, sz, flags))
     return (void *)-1;
-  return REAL(mmap)(addr, sz, prot, flags, fd, off);
+  void *result = REAL(mmap)(addr, sz, prot, flags, fd, off);
+  return (void *)checkMmapResult((uptr)result, sz);
 }
 
 #if SANITIZER_LINUX
@@ -379,7 +343,8 @@ INTERCEPTOR(void *, mmap64, void *addr, SIZE_T sz, int prot, int flags,
                  int fd, OFF64_T off) {
   if (!fixMmapAddr(&addr, sz, flags))
     return (void *)-1;
-  return REAL(mmap64)(addr, sz, prot, flags, fd, off);
+  void *result = REAL(mmap64)(addr, sz, prot, flags, fd, off);
+  return (void *)checkMmapResult((uptr)result, sz);
 }
 #define ESAN_MAYBE_INTERCEPT_MMAP64 INTERCEPT_FUNCTION(mmap64)
 #else
diff --git a/lib/esan/esan_linux.cpp b/lib/esan/esan_linux.cpp
new file mode 100644
index 000000000..aa961b661
--- /dev/null
+++ b/lib/esan/esan_linux.cpp
@@ -0,0 +1,83 @@
+//===-- esan.cpp ----------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of EfficiencySanitizer, a family of performance tuners.
+//
+// Linux-specific code for the Esan run-time.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if SANITIZER_FREEBSD || SANITIZER_LINUX
+
+#include "esan.h"
+#include "esan_shadow.h"
+#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include <sys/mman.h>
+#include <errno.h>
+
+namespace __esan {
+
+void verifyAddressSpace() {
+#if SANITIZER_LINUX && defined(__x86_64__)
+  // The kernel determines its mmap base from the stack size limit.
+  // Our Linux 64-bit shadow mapping assumes the stack limit is less than a
+  // terabyte, which keeps the mmap region above 0x7e00'.
+  uptr StackLimit = GetStackSizeLimitInBytes();
+  if (StackSizeIsUnlimited() || StackLimit > MaxStackSize) {
+    VReport(1, "The stack size limit is beyond the maximum supported.\n"
+            "Re-execing with a stack size below 1TB.\n");
+    SetStackSizeLimitInBytes(MaxStackSize);
+    ReExec();
+  }
+#endif
+}
+
+static bool liesWithinSingleAppRegion(uptr Start, SIZE_T Size) {
+  uptr AppStart, AppEnd;
+  for (int i = 0; getAppRegion(i, &AppStart, &AppEnd); ++i) {
+    if (Start >= AppStart && Start + Size - 1 <= AppEnd) {
+      return true;
+    }
+  }
+  return false;
+}
+
+bool fixMmapAddr(void **Addr, SIZE_T Size, int Flags) {
+  if (*Addr) {
+    if (!liesWithinSingleAppRegion((uptr)*Addr, Size)) {
+      VPrintf(1, "mmap conflict: [%p-%p) is not in an app region\n",
+              *Addr, (uptr)*Addr + Size);
+      if (Flags & MAP_FIXED) {
+        errno = EINVAL;
+        return false;
+      } else {
+        *Addr = 0;
+      }
+    }
+  }
+  return true;
+}
+
+uptr checkMmapResult(uptr Addr, SIZE_T Size) {
+  if ((void *)Addr == MAP_FAILED)
+    return Addr;
+  if (!liesWithinSingleAppRegion(Addr, Size)) {
+    // FIXME: attempt to dynamically add this as an app region if it
+    // fits our shadow criteria.
+    // We could also try to remap somewhere else.
+    Printf("ERROR: unsupported mapping at [%p-%p)\n", Addr, Addr+Size);
+    Die();
+  }
+  return Addr;
+}
+
+} // namespace __esan
+
+#endif // SANITIZER_FREEBSD || SANITIZER_LINUX
diff --git a/lib/esan/esan_shadow.h b/lib/esan/esan_shadow.h
index 4507c3d14..f8f154ef7 100644
--- a/lib/esan/esan_shadow.h
+++ b/lib/esan/esan_shadow.h
@@ -40,6 +40,10 @@ namespace __esan {
 // library region to distinguish the vsyscall's shadow, considering this gap to
 // be an invalid app region.
 // We disallow application memory outside of those 5 regions.
+// Our regions assume that the stack rlimit is less than a terabyte (otherwise
+// the Linux kernel's default mmap region drops below 0x7e00'), which we enforce
+// at init time (we can support larger and unlimited sizes for shadow
+// scaledowns, but it is difficult for 1:1 mappings).
 //
 // Our shadow memory is scaled from a 1:1 mapping and supports a scale
 // specified at library initialization time that can be any power-of-2
@@ -103,6 +107,10 @@ static const struct ApplicationRegion AppRegions[] = {
 };
 static const u32 NumAppRegions = sizeof(AppRegions)/sizeof(AppRegions[0]);
 
+// See the comment above: we do not currently support a stack size rlimit
+// equal to or larger than 1TB.
+static const uptr MaxStackSize = (1ULL << 40) - 4096;
+
 class ShadowMapping {
 public:
   static const uptr Mask = 0x00000fffffffffffu;