[asan] make use-after-return handle very deep recursion; fixes 483.xalancbmk in UAR mode

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@189929 91177308-0d34-0410-b5e6-96231b3b80d8

3 files changed, 32 insertions, 7 deletions

diff --git a/lib/asan/asan_fake_stack.cc b/lib/asan/asan_fake_stack.cc
index 02f92c1f2..62a1982a7 100644
--- a/lib/asan/asan_fake_stack.cc
+++ b/lib/asan/asan_fake_stack.cc
@@ -102,10 +102,12 @@ void FakeStack::AllocateOneSizeClass(uptr size_class) {
   CHECK(ClassMmapSize(size_class) >= GetPageSizeCached());
   uptr new_mem = (uptr)MmapOrDie(
       ClassMmapSize(size_class), __FUNCTION__);
-  // Printf("T%d new_mem[%zu]: %p-%p mmap %zu\n",
-  //       GetCurrentThread()->tid(),
-  //       size_class, new_mem, new_mem + ClassMmapSize(size_class),
-  //       ClassMmapSize(size_class));
+  if (0) {
+    Printf("T%d new_mem[%zu]: %p-%p mmap %zu\n",
+           GetCurrentThread()->tid(),
+           size_class, new_mem, new_mem + ClassMmapSize(size_class),
+           ClassMmapSize(size_class));
+  }
   uptr i;
   uptr size = ClassSize(size_class);
   for (i = 0; i + size <= ClassMmapSize(size_class); i += size) {
diff --git a/lib/asan/asan_fake_stack.h b/lib/asan/asan_fake_stack.h
index 6a5e5f632..986aed7b3 100644
--- a/lib/asan/asan_fake_stack.h
+++ b/lib/asan/asan_fake_stack.h
@@ -81,12 +81,12 @@ class FakeStack {
   uptr StackSize() const { return stack_size_; }
 
  private:
-  static const uptr kMinStackFrameSizeLog = 9;  // Min frame is 512B.
+  static const uptr kMinStackFrameSizeLog = 7;  // Min frame is 128B.
   static const uptr kMaxStackFrameSizeLog = 16;  // Max stack frame is 64K.
   static const uptr kMaxStackMallocSize = 1 << kMaxStackFrameSizeLog;
   static const uptr kNumberOfSizeClasses =
       kMaxStackFrameSizeLog - kMinStackFrameSizeLog + 1;
-  static const uptr kMaxRecursionDepth = 15000;
+  static const uptr kMaxRecursionDepth = 60000;
 
   bool AddrIsInSizeClass(uptr addr, uptr size_class);
 
@@ -110,7 +110,7 @@ class FakeStack {
   FakeFrameLifo<kMaxRecursionDepth> call_stack_;
 };
 
-COMPILER_CHECK(sizeof(FakeStack) <= (1 << 17));
+COMPILER_CHECK(sizeof(FakeStack) <= (1 << 19));
 
 }  // namespace __asan
 
diff --git a/lib/asan/lit_tests/TestCases/deep_call_stack.cc b/lib/asan/lit_tests/TestCases/deep_call_stack.cc
new file mode 100644
index 000000000..840ea3872
--- /dev/null
+++ b/lib/asan/lit_tests/TestCases/deep_call_stack.cc
@@ -0,0 +1,23 @@
+// Check that UAR mode can handle very deep recusrion.
+//
+// RUN: %clangxx_asan -fsanitize=use-after-return -O2 %s -o %t && \
+// RUN:   %t 2>&1 | FileCheck %s
+#include <stdio.h>
+
+__attribute__((noinline))
+void RecursiveFunc(int depth, int *ptr) {
+  if ((depth % 1000) == 0)
+    printf("[%05d] ptr: %p\n", depth, ptr);
+  if (depth == 0)
+    return;
+  int local;
+  RecursiveFunc(depth - 1, &local);
+}
+
+int main(int argc, char **argv) {
+  RecursiveFunc(40000, 0);
+  return 0;
+}
+// CHECK: [40000] ptr:
+// CHECK: [20000] ptr:
+// CHECK: [00000] ptr