[esan] Add handling of large stack size rlimits

Summary:
Adds detection of large stack size rlimits (over 1 TB or unlimited), which
results in an mmap location that our shadow mapping does not support.  We
re-exec the application in this situation.  Adds a test of this behavior.

Adds general detection of mmap regions outside of our app regions.  In the
future we want to try to adaptively handle these but for now we abort.

Moves the existing Linux-specific mmap code into a platform-specific file
where the new rlimit code lives.

Reviewers: eugenis

Subscribers: vitalybuka, zhaoqin, kcc, aizatsky, llvm-commits, kubabrecka

Differential Revision: http://reviews.llvm.org/D20745

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@271079 91177308-0d34-0410-b5e6-96231b3b80d8

4 files changed, 82 insertions, 5 deletions

diff --git a/test/esan/TestCases/large-stack-linux.c b/test/esan/TestCases/large-stack-linux.c
new file mode 100644
index 000000000..856da2a1b
--- /dev/null
+++ b/test/esan/TestCases/large-stack-linux.c
@@ -0,0 +1,74 @@
+// RUN: %clang_esan_wset -O0 %s -o %t 2>&1
+// RUN: %env_esan_opts=verbosity=1 %run %t %t 2>&1 | FileCheck %s
+
+#include <assert.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <sys/resource.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+static void testChildStackLimit(rlim_t StackLimit, char *ToRun) {
+  int Res;
+  struct rlimit Limit;
+  Limit.rlim_cur = RLIM_INFINITY;
+  Limit.rlim_max = RLIM_INFINITY;
+  Res = setrlimit(RLIMIT_STACK, &Limit);
+  if (Res != 0) {
+    // Probably our environment had a large limit and we ourselves got
+    // re-execed and can no longer raise our limit.
+    // We have to bail and emulate the regular test.
+    // We'd prefer to have branches in our FileCheck output to ensure the
+    // initial program was re-execed but this is the best we can do for now.
+    fprintf(stderr, "in esan::initializeLibrary\n");
+    fprintf(stderr, "==1234==The stack size limit is beyond the maximum supported.\n");
+    fprintf(stderr, "Re-execing with a stack size below 1TB.\n");
+    fprintf(stderr, "in esan::initializeLibrary\n");
+    fprintf(stderr, "done\n");
+    fprintf(stderr, "in esan::finalizeLibrary\n");
+    return;
+  }
+
+  pid_t Child = fork();
+  assert(Child >= 0);
+  if (Child > 0) {
+    pid_t WaitRes = waitpid(Child, NULL, 0);
+    assert(WaitRes == Child);
+  } else {
+    char *Args[2];
+    Args[0] = ToRun;
+    Args[1] = NULL;
+    Res = execv(ToRun, Args);
+    assert(0); // Should not be reached.
+  }
+}
+
+int main(int argc, char *argv[]) {
+  // The path to the program to exec must be passed in the first time.
+  if (argc == 2) {
+    fprintf(stderr, "Testing child with infinite stack\n");
+    testChildStackLimit(RLIM_INFINITY, argv[1]);
+    fprintf(stderr, "Testing child with 1TB stack\n");
+    testChildStackLimit(1ULL << 40, argv[1]);
+  }
+  fprintf(stderr, "done\n");
+  // CHECK:      in esan::initializeLibrary
+  // CHECK:      Testing child with infinite stack
+  // CHECK-NEXT: in esan::initializeLibrary
+  // CHECK-NEXT: =={{[0-9]+}}==The stack size limit is beyond the maximum supported.
+  // CHECK-NEXT: Re-execing with a stack size below 1TB.
+  // CHECK-NEXT: in esan::initializeLibrary
+  // CHECK:      done
+  // CHECK:      in esan::finalizeLibrary
+  // CHECK:      Testing child with 1TB stack
+  // CHECK-NEXT: in esan::initializeLibrary
+  // CHECK-NEXT: =={{[0-9]+}}==The stack size limit is beyond the maximum supported.
+  // CHECK-NEXT: Re-execing with a stack size below 1TB.
+  // CHECK-NEXT: in esan::initializeLibrary
+  // CHECK:      done
+  // CHECK-NEXT: in esan::finalizeLibrary
+  // CHECK:      done
+  // CHECK-NEXT: in esan::finalizeLibrary
+  return 0;
+}
diff --git a/test/esan/TestCases/mmap-shadow-conflict.c b/test/esan/TestCases/mmap-shadow-conflict.c
index cb45bd1fe..4b3c58b06 100644
--- a/test/esan/TestCases/mmap-shadow-conflict.c
+++ b/test/esan/TestCases/mmap-shadow-conflict.c
@@ -16,7 +16,8 @@ int main(int argc, char **argv) {
                    MAP_ANON|MAP_PRIVATE, -1, 0);
   fprintf(stderr, "mapped %p\n", Map);
   // CHECK:      in esan::initializeLibrary
-  // CHECK-NEXT: Shadow scale=2 offset=0x440000000000
+  // (There can be a re-exec for stack limit here.)
+  // CHECK:      Shadow scale=2 offset=0x440000000000
   // CHECK-NEXT: Shadow #0: [110000000000-114000000000) (256GB)
   // CHECK-NEXT: Shadow #1: [124000000000-12c000000000) (512GB)
   // CHECK-NEXT: Shadow #2: [148000000000-150000000000) (512GB)
diff --git a/test/esan/TestCases/verbose-simple.c b/test/esan/TestCases/verbose-simple.c
index e793f08a9..c136dc49f 100644
--- a/test/esan/TestCases/verbose-simple.c
+++ b/test/esan/TestCases/verbose-simple.c
@@ -3,7 +3,8 @@
 
 int main(int argc, char **argv) {
   // CHECK:      in esan::initializeLibrary
-  // CHECK-NEXT: Shadow scale=2 offset=0x440000000000
+  // (There can be a re-exec for stack limit here.)
+  // CHECK:      Shadow scale=2 offset=0x440000000000
   // CHECK-NEXT: Shadow #0: [110000000000-114000000000) (256GB)
   // CHECK-NEXT: Shadow #1: [124000000000-12c000000000) (512GB)
   // CHECK-NEXT: Shadow #2: [148000000000-150000000000) (512GB)
diff --git a/test/esan/TestCases/workingset-memset.cpp b/test/esan/TestCases/workingset-memset.cpp
index 0c5db6935..a0c36e3ae 100644
--- a/test/esan/TestCases/workingset-memset.cpp
+++ b/test/esan/TestCases/workingset-memset.cpp
@@ -8,12 +8,13 @@
 #include <string.h>
 
 int main(int argc, char **argv) {
-  const int iters = 630;
-  const int size = 64*iters;
-  char *p = (char *)malloc(size);
+  const int size = 128*1024*1024;
+  char *p = (char *)mmap(0, size, PROT_READ | PROT_WRITE,
+                         MAP_ANON | MAP_PRIVATE, -1, 0);
   // Test the slowpath at different cache line boundaries.
   for (int i = 0; i < 630; i++)
     memset((char *)p + 63*i, i, 63*i);
+  munmap(p, size);
   return 0;
   // FIXME: once the memory scan and size report is in place add it here.
   // CHECK: {{.*}}EfficiencySanitizer is not finished: nothing yet to report