[TSan] Report proper error on allocator failures instead of CHECK(0)-ing

Summary:
Following up on and complementing D44404 and other sanitizer allocators.

Currently many allocator specific errors (OOM, for example) are reported as
a text message and CHECK(0) termination, no stack, no details, not too
helpful nor informative. To improve the situation, detailed and structured
common errors were defined and reported under the appropriate conditions.

Common tests were generalized a bit to cover a slightly different TSan
stack reporting format, extended to verify errno value and returned
pointer value check is now explicit to facilitate debugging.

Reviewers: dvyukov

Subscribers: srhines, kubamracek, delcypher, #sanitizers, llvm-commits

Differential Revision: https://reviews.llvm.org/D48087

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@334975 91177308-0d34-0410-b5e6-96231b3b80d8

6 files changed, 67 insertions, 53 deletions

diff --git a/lib/tsan/rtl/tsan_mman.cc b/lib/tsan/rtl/tsan_mman.cc
index 39c0d8607..b160a9736 100644
--- a/lib/tsan/rtl/tsan_mman.cc
+++ b/lib/tsan/rtl/tsan_mman.cc
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 #include "sanitizer_common/sanitizer_allocator_checks.h"
 #include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_allocator_report.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_errno.h"
 #include "sanitizer_common/sanitizer_placement_new.h"
@@ -150,13 +151,24 @@ static void SignalUnsafeCall(ThreadState *thr, uptr pc) {
   OutputReport(thr, rep);
 }
 
+static constexpr uptr kMaxAllowedMallocSize = 1ull << 40;
+
 void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz, uptr align,
                           bool signal) {
-  if ((sz >= (1ull << 40)) || (align >= (1ull << 40)))
-    return ReturnNullOrDieOnFailure::OnBadRequest();
+  if (sz >= kMaxAllowedMallocSize || align >= kMaxAllowedMallocSize) {
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportAllocationSizeTooBig(sz, kMaxAllowedMallocSize, &stack);
+  }
   void *p = allocator()->Allocate(&thr->proc()->alloc_cache, sz, align);
-  if (UNLIKELY(p == 0))
-    return ReturnNullOrDieOnFailure::OnOOM();
+  if (UNLIKELY(!p)) {
+    SetAllocatorOutOfMemory();
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportOutOfMemory(sz, &stack);
+  }
   if (ctx && ctx->initialized)
     OnUserAlloc(thr, pc, (uptr)p, sz, true);
   if (signal)
@@ -178,8 +190,12 @@ void *user_alloc(ThreadState *thr, uptr pc, uptr sz) {
 }
 
 void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) {
-  if (UNLIKELY(CheckForCallocOverflow(size, n)))
-    return SetErrnoOnNull(ReturnNullOrDieOnFailure::OnBadRequest());
+  if (UNLIKELY(CheckForCallocOverflow(size, n))) {
+    if (AllocatorMayReturnNull())
+      return SetErrnoOnNull(nullptr);
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportCallocOverflow(n, size, &stack);
+  }
   void *p = user_alloc_internal(thr, pc, n * size);
   if (p)
     internal_memset(p, 0, n * size);
@@ -224,7 +240,10 @@ void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz) {
 void *user_memalign(ThreadState *thr, uptr pc, uptr align, uptr sz) {
   if (UNLIKELY(!IsPowerOfTwo(align))) {
     errno = errno_EINVAL;
-    return ReturnNullOrDieOnFailure::OnBadRequest();
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportInvalidAllocationAlignment(align, &stack);
   }
   return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, align));
 }
@@ -232,11 +251,14 @@ void *user_memalign(ThreadState *thr, uptr pc, uptr align, uptr sz) {
 int user_posix_memalign(ThreadState *thr, uptr pc, void **memptr, uptr align,
                         uptr sz) {
   if (UNLIKELY(!CheckPosixMemalignAlignment(align))) {
-    ReturnNullOrDieOnFailure::OnBadRequest();
-    return errno_EINVAL;
+    if (AllocatorMayReturnNull())
+      return errno_EINVAL;
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportInvalidPosixMemalignAlignment(align, &stack);
   }
   void *ptr = user_alloc_internal(thr, pc, sz, align);
   if (UNLIKELY(!ptr))
+    // OOM error is already taken care of by user_alloc_internal.
     return errno_ENOMEM;
   CHECK(IsAligned((uptr)ptr, align));
   *memptr = ptr;
@@ -246,7 +268,10 @@ int user_posix_memalign(ThreadState *thr, uptr pc, void **memptr, uptr align,
 void *user_aligned_alloc(ThreadState *thr, uptr pc, uptr align, uptr sz) {
   if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(align, sz))) {
     errno = errno_EINVAL;
-    return ReturnNullOrDieOnFailure::OnBadRequest();
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportInvalidAlignedAllocAlignment(sz, align, &stack);
   }
   return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, align));
 }
@@ -259,7 +284,10 @@ void *user_pvalloc(ThreadState *thr, uptr pc, uptr sz) {
   uptr PageSize = GetPageSizeCached();
   if (UNLIKELY(CheckForPvallocOverflow(sz, PageSize))) {
     errno = errno_ENOMEM;
-    return ReturnNullOrDieOnFailure::OnBadRequest();
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportPvallocOverflow(sz, &stack);
   }
   // pvalloc(0) should allocate one page.
   sz = sz ? RoundUpTo(sz, PageSize) : PageSize;
diff --git a/lib/tsan/rtl/tsan_new_delete.cc b/lib/tsan/rtl/tsan_new_delete.cc
index a1bb22690..4f52d3d71 100644
--- a/lib/tsan/rtl/tsan_new_delete.cc
+++ b/lib/tsan/rtl/tsan_new_delete.cc
@@ -13,8 +13,10 @@
 //===----------------------------------------------------------------------===//
 #include "interception/interception.h"
 #include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_report.h"
 #include "sanitizer_common/sanitizer_internal_defs.h"
 #include "tsan_interceptors.h"
+#include "tsan_rtl.h"
 
 using namespace __tsan;  // NOLINT
 
@@ -34,7 +36,10 @@ DECLARE_REAL(void, free, void *ptr)
   {  \
     SCOPED_INTERCEPTOR_RAW(mangled_name, size); \
     p = user_alloc(thr, pc, size); \
-    if (!nothrow && UNLIKELY(!p)) DieOnFailure::OnOOM(); \
+    if (!nothrow && UNLIKELY(!p)) { \
+      GET_STACK_TRACE_FATAL(thr, pc); \
+      ReportOutOfMemory(size, &stack); \
+    } \
   }  \
   invoke_malloc_hook(p, size);  \
   return p;
@@ -46,7 +51,10 @@ DECLARE_REAL(void, free, void *ptr)
   {  \
     SCOPED_INTERCEPTOR_RAW(mangled_name, size); \
     p = user_memalign(thr, pc, (uptr)align, size); \
-    if (!nothrow && UNLIKELY(!p)) DieOnFailure::OnOOM(); \
+    if (!nothrow && UNLIKELY(!p)) { \
+      GET_STACK_TRACE_FATAL(thr, pc); \
+      ReportOutOfMemory(size, &stack); \
+    } \
   }  \
   invoke_malloc_hook(p, size);  \
   return p;
diff --git a/lib/tsan/rtl/tsan_rtl.h b/lib/tsan/rtl/tsan_rtl.h
index 5e2a745c9..523b69aaa 100644
--- a/lib/tsan/rtl/tsan_rtl.h
+++ b/lib/tsan/rtl/tsan_rtl.h
@@ -650,6 +650,10 @@ void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack,
   ExtractTagFromStack(stack, tag);
 }
 
+#define GET_STACK_TRACE_FATAL(thr, pc) \
+  VarSizeStackTrace stack; \
+  ObtainCurrentStack(thr, pc, &stack); \
+  stack.ReverseOrder();
 
 #if TSAN_COLLECT_STATS
 void StatAggregate(u64 *dst, u64 *src);
diff --git a/lib/tsan/rtl/tsan_stack_trace.cc b/lib/tsan/rtl/tsan_stack_trace.cc
index ceca3f8e8..a0dee19e2 100644
--- a/lib/tsan/rtl/tsan_stack_trace.cc
+++ b/lib/tsan/rtl/tsan_stack_trace.cc
@@ -43,4 +43,9 @@ void VarSizeStackTrace::Init(const uptr *pcs, uptr cnt, uptr extra_top_pc) {
     trace_buffer[cnt] = extra_top_pc;
 }
 
+void VarSizeStackTrace::ReverseOrder() {
+  for (u32 i = 0; i < (size >> 1); i++)
+    Swap(trace_buffer[i], trace_buffer[size - 1 - i]);
+}
+
 }  // namespace __tsan
diff --git a/lib/tsan/rtl/tsan_stack_trace.h b/lib/tsan/rtl/tsan_stack_trace.h
index 5bf89bb75..f69b57464 100644
--- a/lib/tsan/rtl/tsan_stack_trace.h
+++ b/lib/tsan/rtl/tsan_stack_trace.h
@@ -27,6 +27,10 @@ struct VarSizeStackTrace : public StackTrace {
   ~VarSizeStackTrace();
   void Init(const uptr *pcs, uptr cnt, uptr extra_top_pc = 0);
 
+  // Reverses the current stack trace order, the top frame goes to the bottom,
+  // the last frame goes to the top.
+  void ReverseOrder();
+
  private:
   void ResizeBuffer(uptr new_size);
 
diff --git a/lib/tsan/tests/unit/tsan_mman_test.cc b/lib/tsan/tests/unit/tsan_mman_test.cc
index 05ae42867..26e13a55c 100644
--- a/lib/tsan/tests/unit/tsan_mman_test.cc
+++ b/lib/tsan/tests/unit/tsan_mman_test.cc
@@ -153,29 +153,12 @@ TEST(Mman, Valloc) {
   EXPECT_NE(p, (void*)0);
   EXPECT_EQ(page_size, __sanitizer_get_allocated_size(p));
   user_free(thr, 0, p);
-
-  EXPECT_DEATH(p = user_pvalloc(thr, 0, (uptr)-(page_size - 1)),
-               "allocator is terminating the process instead of returning 0");
-  EXPECT_DEATH(p = user_pvalloc(thr, 0, (uptr)-1),
-               "allocator is terminating the process instead of returning 0");
 }
 
 #if !SANITIZER_DEBUG
 // EXPECT_DEATH clones a thread with 4K stack,
 // which is overflown by tsan memory accesses functions in debug mode.
 
-TEST(Mman, CallocOverflow) {
-  ThreadState *thr = cur_thread();
-  uptr pc = 0;
-  size_t kArraySize = 4096;
-  volatile size_t kMaxSizeT = std::numeric_limits<size_t>::max();
-  volatile size_t kArraySize2 = kMaxSizeT / kArraySize + 10;
-  volatile void *p = NULL;
-  EXPECT_DEATH(p = user_calloc(thr, pc, kArraySize, kArraySize2),
-               "allocator is terminating the process instead of returning 0");
-  EXPECT_EQ(0L, p);
-}
-
 TEST(Mman, Memalign) {
   ThreadState *thr = cur_thread();
 
@@ -183,12 +166,16 @@ TEST(Mman, Memalign) {
   EXPECT_NE(p, (void*)0);
   user_free(thr, 0, p);
 
+  // TODO(alekseyshl): Remove this death test when memalign is verified by
+  // tests in sanitizer_common.
   p = NULL;
   EXPECT_DEATH(p = user_memalign(thr, 0, 7, 100),
-               "allocator is terminating the process instead of returning 0");
+               "invalid-allocation-alignment");
   EXPECT_EQ(0L, p);
 }
 
+#endif
+
 TEST(Mman, PosixMemalign) {
   ThreadState *thr = cur_thread();
 
@@ -197,16 +184,6 @@ TEST(Mman, PosixMemalign) {
   EXPECT_NE(p, (void*)0);
   EXPECT_EQ(res, 0);
   user_free(thr, 0, p);
-
-  p = NULL;
-  // Alignment is not a power of two, although is a multiple of sizeof(void*).
-  EXPECT_DEATH(res = user_posix_memalign(thr, 0, &p, 3 * sizeof(p), 100),
-               "allocator is terminating the process instead of returning 0");
-  EXPECT_EQ(0L, p);
-  // Alignment is not a multiple of sizeof(void*), although is a power of 2.
-  EXPECT_DEATH(res = user_posix_memalign(thr, 0, &p, 2, 100),
-               "allocator is terminating the process instead of returning 0");
-  EXPECT_EQ(0L, p);
 }
 
 TEST(Mman, AlignedAlloc) {
@@ -215,18 +192,6 @@ TEST(Mman, AlignedAlloc) {
   void *p = user_aligned_alloc(thr, 0, 8, 64);
   EXPECT_NE(p, (void*)0);
   user_free(thr, 0, p);
-
-  p = NULL;
-  // Alignement is not a power of 2.
-  EXPECT_DEATH(p = user_aligned_alloc(thr, 0, 7, 100),
-               "allocator is terminating the process instead of returning 0");
-  EXPECT_EQ(0L, p);
-  // Size is not a multiple of alignment.
-  EXPECT_DEATH(p = user_aligned_alloc(thr, 0, 8, 100),
-               "allocator is terminating the process instead of returning 0");
-  EXPECT_EQ(0L, p);
 }
 
-#endif
-
 }  // namespace __tsan