[Sanitizers] Operator new() interceptors always die on allocation error

Summary:
Operator new interceptors behavior is now controlled by their nothrow
property as well as by allocator_may_return_null flag value:

- allocator_may_return_null=* + new()        - die on allocation error
- allocator_may_return_null=0 + new(nothrow) - die on allocation error
- allocator_may_return_null=1 + new(nothrow) - return null

Ideally new() should throw std::bad_alloc exception, but that is not
trivial to achieve, hence TODO.

Reviewers: eugenis

Subscribers: kubamracek, llvm-commits

Differential Revision: https://reviews.llvm.org/D34731

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@306604 91177308-0d34-0410-b5e6-96231b3b80d8

8 files changed, 78 insertions, 45 deletions

diff --git a/lib/asan/asan_allocator.cc b/lib/asan/asan_allocator.cc
index 57651f49c..36348834e 100644
--- a/lib/asan/asan_allocator.cc
+++ b/lib/asan/asan_allocator.cc
@@ -160,7 +160,11 @@ struct QuarantineCallback {
   }
 
   void *Allocate(uptr size) {
-    return get_allocator().Allocate(cache_, size, 1);
+    void *res = get_allocator().Allocate(cache_, size, 1);
+    // TODO(alekseys): Consider making quarantine OOM-friendly.
+    if (UNLIKELY(!res))
+      return DieOnFailure::OnOOM();
+    return res;
   }
 
   void Deallocate(void *p) {
@@ -524,8 +528,7 @@ struct Allocator {
 
   // Expects the chunk to already be marked as quarantined by using
   // AtomicallySetQuarantineFlagIfAllocated.
-  void QuarantineChunk(AsanChunk *m, void *ptr, BufferedStackTrace *stack,
-                       AllocType alloc_type) {
+  void QuarantineChunk(AsanChunk *m, void *ptr, BufferedStackTrace *stack) {
     CHECK_EQ(m->chunk_state, CHUNK_QUARANTINE);
     CHECK_GE(m->alloc_tid, 0);
     if (SANITIZER_WORDSIZE == 64)  // On 32-bits this resides in user area.
@@ -603,7 +606,7 @@ struct Allocator {
       ReportNewDeleteSizeMismatch(p, delete_size, stack);
     }
 
-    QuarantineChunk(m, ptr, stack, alloc_type);
+    QuarantineChunk(m, ptr, stack);
   }
 
   void *Reallocate(void *old_ptr, uptr new_size, BufferedStackTrace *stack) {
diff --git a/lib/asan/asan_new_delete.cc b/lib/asan/asan_new_delete.cc
index 3283fb394..a64fe583b 100644
--- a/lib/asan/asan_new_delete.cc
+++ b/lib/asan/asan_new_delete.cc
@@ -63,12 +63,17 @@ struct nothrow_t {};
 enum class align_val_t: size_t {};
 }  // namespace std
 
-#define OPERATOR_NEW_BODY(type) \
+// TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
+#define OPERATOR_NEW_BODY(type, nothrow) \
   GET_STACK_TRACE_MALLOC;\
-  return asan_memalign(0, size, &stack, type);
-#define OPERATOR_NEW_BODY_ALIGN(type) \
+  void *res = asan_memalign(0, size, &stack, type);\
+  if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();\
+  return res;
+#define OPERATOR_NEW_BODY_ALIGN(type, nothrow) \
   GET_STACK_TRACE_MALLOC;\
-  return asan_memalign((uptr)align, size, &stack, type);
+  void *res = asan_memalign((uptr)align, size, &stack, type);\
+  if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();\
+  return res;
 
 // On OS X it's not enough to just provide our own 'operator new' and
 // 'operator delete' implementations, because they're going to be in the
@@ -79,40 +84,42 @@ enum class align_val_t: size_t {};
 // OS X we need to intercept them using their mangled names.
 #if !SANITIZER_MAC
 CXX_OPERATOR_ATTRIBUTE
-void *operator new(size_t size) { OPERATOR_NEW_BODY(FROM_NEW); }
+void *operator new(size_t size)
+{ OPERATOR_NEW_BODY(FROM_NEW, false /*nothrow*/); }
 CXX_OPERATOR_ATTRIBUTE
-void *operator new[](size_t size) { OPERATOR_NEW_BODY(FROM_NEW_BR); }
+void *operator new[](size_t size)
+{ OPERATOR_NEW_BODY(FROM_NEW_BR, false /*nothrow*/); }
 CXX_OPERATOR_ATTRIBUTE
 void *operator new(size_t size, std::nothrow_t const&)
-{ OPERATOR_NEW_BODY(FROM_NEW); }
+{ OPERATOR_NEW_BODY(FROM_NEW, true /*nothrow*/); }
 CXX_OPERATOR_ATTRIBUTE
 void *operator new[](size_t size, std::nothrow_t const&)
-{ OPERATOR_NEW_BODY(FROM_NEW_BR); }
+{ OPERATOR_NEW_BODY(FROM_NEW_BR, true /*nothrow*/); }
 CXX_OPERATOR_ATTRIBUTE
 void *operator new(size_t size, std::align_val_t align)
-{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW); }
+{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW, false /*nothrow*/); }
 CXX_OPERATOR_ATTRIBUTE
 void *operator new[](size_t size, std::align_val_t align)
-{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW_BR); }
+{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW_BR, false /*nothrow*/); }
 CXX_OPERATOR_ATTRIBUTE
 void *operator new(size_t size, std::align_val_t align, std::nothrow_t const&)
-{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW); }
+{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW, true /*nothrow*/); }
 CXX_OPERATOR_ATTRIBUTE
 void *operator new[](size_t size, std::align_val_t align, std::nothrow_t const&)
-{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW_BR); }
+{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW_BR, true /*nothrow*/); }
 
 #else  // SANITIZER_MAC
 INTERCEPTOR(void *, _Znwm, size_t size) {
-  OPERATOR_NEW_BODY(FROM_NEW);
+  OPERATOR_NEW_BODY(FROM_NEW, false /*nothrow*/);
 }
 INTERCEPTOR(void *, _Znam, size_t size) {
-  OPERATOR_NEW_BODY(FROM_NEW_BR);
+  OPERATOR_NEW_BODY(FROM_NEW_BR, false /*nothrow*/);
 }
 INTERCEPTOR(void *, _ZnwmRKSt9nothrow_t, size_t size, std::nothrow_t const&) {
-  OPERATOR_NEW_BODY(FROM_NEW);
+  OPERATOR_NEW_BODY(FROM_NEW, true /*nothrow*/);
 }
 INTERCEPTOR(void *, _ZnamRKSt9nothrow_t, size_t size, std::nothrow_t const&) {
-  OPERATOR_NEW_BODY(FROM_NEW_BR);
+  OPERATOR_NEW_BODY(FROM_NEW_BR, true /*nothrow*/);
 }
 #endif
 
diff --git a/lib/lsan/lsan_interceptors.cc b/lib/lsan/lsan_interceptors.cc
index 9e39a7d19..b6ac7b77c 100644
--- a/lib/lsan/lsan_interceptors.cc
+++ b/lib/lsan/lsan_interceptors.cc
@@ -199,19 +199,26 @@ INTERCEPTOR(int, mprobe, void *ptr) {
 }
 #endif // SANITIZER_INTERCEPT_MCHECK_MPROBE
 
-#define OPERATOR_NEW_BODY                              \
-  ENSURE_LSAN_INITED;                                  \
-  GET_STACK_TRACE_MALLOC;                              \
-  return Allocate(stack, size, 1, kAlwaysClearMemory);
+// TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
+#define OPERATOR_NEW_BODY(nothrow)                         \
+  ENSURE_LSAN_INITED;                                      \
+  GET_STACK_TRACE_MALLOC;                                  \
+  void *res = Allocate(stack, size, 1, kAlwaysClearMemory);\
+  if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();\
+  return res;
 
 INTERCEPTOR_ATTRIBUTE
-void *operator new(size_t size) { OPERATOR_NEW_BODY; }
+void *operator new(size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); }
 INTERCEPTOR_ATTRIBUTE
-void *operator new[](size_t size) { OPERATOR_NEW_BODY; }
+void *operator new[](size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); }
 INTERCEPTOR_ATTRIBUTE
-void *operator new(size_t size, std::nothrow_t const&) { OPERATOR_NEW_BODY; }
+void *operator new(size_t size, std::nothrow_t const&) {
+  OPERATOR_NEW_BODY(true /*nothrow*/);
+}
 INTERCEPTOR_ATTRIBUTE
-void *operator new[](size_t size, std::nothrow_t const&) { OPERATOR_NEW_BODY; }
+void *operator new[](size_t size, std::nothrow_t const&) {
+  OPERATOR_NEW_BODY(true /*nothrow*/);
+}
 
 #define OPERATOR_DELETE_BODY \
   ENSURE_LSAN_INITED;        \
diff --git a/lib/msan/msan_new_delete.cc b/lib/msan/msan_new_delete.cc
index 540100316..c7295feeb 100644
--- a/lib/msan/msan_new_delete.cc
+++ b/lib/msan/msan_new_delete.cc
@@ -14,6 +14,7 @@
 
 #include "msan.h"
 #include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator.h"
 
 #if MSAN_REPLACE_OPERATORS_NEW_AND_DELETE
 
@@ -27,18 +28,25 @@ namespace std {
 }  // namespace std
 
 
-#define OPERATOR_NEW_BODY \
+// TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
+#define OPERATOR_NEW_BODY(nothrow) \
   GET_MALLOC_STACK_TRACE; \
-  return MsanReallocate(&stack, 0, size, sizeof(u64), false)
+  void *res = MsanReallocate(&stack, 0, size, sizeof(u64), false);\
+  if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();\
+  return res
 
 INTERCEPTOR_ATTRIBUTE
-void *operator new(size_t size) { OPERATOR_NEW_BODY; }
+void *operator new(size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); }
 INTERCEPTOR_ATTRIBUTE
-void *operator new[](size_t size) { OPERATOR_NEW_BODY; }
+void *operator new[](size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); }
 INTERCEPTOR_ATTRIBUTE
-void *operator new(size_t size, std::nothrow_t const&) { OPERATOR_NEW_BODY; }
+void *operator new(size_t size, std::nothrow_t const&) {
+  OPERATOR_NEW_BODY(true /*nothrow*/);
+}
 INTERCEPTOR_ATTRIBUTE
-void *operator new[](size_t size, std::nothrow_t const&) { OPERATOR_NEW_BODY; }
+void *operator new[](size_t size, std::nothrow_t const&) {
+  OPERATOR_NEW_BODY(true /*nothrow*/);
+}
 
 #define OPERATOR_DELETE_BODY \
   GET_MALLOC_STACK_TRACE; \
diff --git a/lib/sanitizer_common/sanitizer_allocator.cc b/lib/sanitizer_common/sanitizer_allocator.cc
index db3ebb033..e081cc576 100644
--- a/lib/sanitizer_common/sanitizer_allocator.cc
+++ b/lib/sanitizer_common/sanitizer_allocator.cc
@@ -246,11 +246,11 @@ void *ReturnNullOrDieOnFailure::OnOOM() {
   ReportAllocatorCannotReturnNull();
 }
 
-void *DieOnFailure::OnBadRequest() {
+void NORETURN *DieOnFailure::OnBadRequest() {
   ReportAllocatorCannotReturnNull();
 }
 
-void *DieOnFailure::OnOOM() {
+void NORETURN *DieOnFailure::OnOOM() {
   atomic_store_relaxed(&allocator_out_of_memory, 1);
   ReportAllocatorCannotReturnNull();
 }
diff --git a/lib/sanitizer_common/sanitizer_allocator.h b/lib/sanitizer_common/sanitizer_allocator.h
index f59c13d1c..06f82d3a2 100644
--- a/lib/sanitizer_common/sanitizer_allocator.h
+++ b/lib/sanitizer_common/sanitizer_allocator.h
@@ -39,8 +39,8 @@ struct ReturnNullOrDieOnFailure {
 };
 // Always dies on the failure.
 struct DieOnFailure {
-  static void *OnBadRequest();
-  static void *OnOOM();
+  static void NORETURN *OnBadRequest();
+  static void NORETURN *OnOOM();
 };
 
 // Returns true if allocator detected OOM condition. Can be used to avoid memory
diff --git a/lib/scudo/scudo_new_delete.cpp b/lib/scudo/scudo_new_delete.cpp
index c022bd0ac..cdefb127b 100644
--- a/lib/scudo/scudo_new_delete.cpp
+++ b/lib/scudo/scudo_new_delete.cpp
@@ -26,13 +26,18 @@ namespace std {
 struct nothrow_t {};
 }  // namespace std
 
+// TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
 CXX_OPERATOR_ATTRIBUTE
 void *operator new(size_t size) {
-  return scudoMalloc(size, FromNew);
+  void *res = scudoMalloc(size, FromNew);
+  if (UNLIKELY(!res)) DieOnFailure::OnOOM();
+  return res;
 }
 CXX_OPERATOR_ATTRIBUTE
 void *operator new[](size_t size) {
-  return scudoMalloc(size, FromNewArray);
+  void *res = scudoMalloc(size, FromNewArray);
+  if (UNLIKELY(!res)) DieOnFailure::OnOOM();
+  return res;
 }
 CXX_OPERATOR_ATTRIBUTE
 void *operator new(size_t size, std::nothrow_t const&) {
diff --git a/lib/tsan/rtl/tsan_new_delete.cc b/lib/tsan/rtl/tsan_new_delete.cc
index b6478bb08..4d03145c1 100644
--- a/lib/tsan/rtl/tsan_new_delete.cc
+++ b/lib/tsan/rtl/tsan_new_delete.cc
@@ -12,6 +12,7 @@
 // Interceptors for operators new and delete.
 //===----------------------------------------------------------------------===//
 #include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator.h"
 #include "sanitizer_common/sanitizer_internal_defs.h"
 #include "tsan_interceptors.h"
 
@@ -24,13 +25,15 @@ struct nothrow_t {};
 DECLARE_REAL(void *, malloc, uptr size)
 DECLARE_REAL(void, free, void *ptr)
 
-#define OPERATOR_NEW_BODY(mangled_name) \
+// TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
+#define OPERATOR_NEW_BODY(mangled_name, nothrow) \
   if (cur_thread()->in_symbolizer) \
     return InternalAlloc(size); \
   void *p = 0; \
   {  \
     SCOPED_INTERCEPTOR_RAW(mangled_name, size); \
     p = user_alloc(thr, pc, size); \
+    if (!nothrow && UNLIKELY(!p)) DieOnFailure::OnOOM(); \
   }  \
   invoke_malloc_hook(p, size);  \
   return p;
@@ -38,25 +41,25 @@ DECLARE_REAL(void, free, void *ptr)
 SANITIZER_INTERFACE_ATTRIBUTE
 void *operator new(__sanitizer::uptr size);
 void *operator new(__sanitizer::uptr size) {
-  OPERATOR_NEW_BODY(_Znwm);
+  OPERATOR_NEW_BODY(_Znwm, false /*nothrow*/);
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void *operator new[](__sanitizer::uptr size);
 void *operator new[](__sanitizer::uptr size) {
-  OPERATOR_NEW_BODY(_Znam);
+  OPERATOR_NEW_BODY(_Znam, false /*nothrow*/);
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void *operator new(__sanitizer::uptr size, std::nothrow_t const&);
 void *operator new(__sanitizer::uptr size, std::nothrow_t const&) {
-  OPERATOR_NEW_BODY(_ZnwmRKSt9nothrow_t);
+  OPERATOR_NEW_BODY(_ZnwmRKSt9nothrow_t, true /*nothrow*/);
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void *operator new[](__sanitizer::uptr size, std::nothrow_t const&);
 void *operator new[](__sanitizer::uptr size, std::nothrow_t const&) {
-  OPERATOR_NEW_BODY(_ZnamRKSt9nothrow_t);
+  OPERATOR_NEW_BODY(_ZnamRKSt9nothrow_t, true /*nothrow*/);
 }
 
 #define OPERATOR_DELETE_BODY(mangled_name) \