[Sanitizers] Move cached allocator_may_return_null flag to sanitizer_allocator

Summary:
Move cached allocator_may_return_null flag to sanitizer_allocator.cc and
provide API to consolidate and unify the behavior of all specific allocators.

Make all sanitizers using CombinedAllocator to follow
AllocatorReturnNullOrDieOnOOM() rules to behave the same way when OOM
happens.

When OOM happens, turn allocator_out_of_memory flag on regardless of
allocator_may_return_null flag value (it used to not to be set when
allocator_may_return_null == true).

release_to_os_interval_ms and rss_limit_exceeded will likely be moved to
sanitizer_allocator.cc too (later).

Reviewers: eugenis

Subscribers: srhines, kubamracek, llvm-commits

Differential Revision: https://reviews.llvm.org/D34310

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@305858 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 40 insertions, 7 deletions

diff --git a/lib/sanitizer_common/sanitizer_allocator.cc b/lib/sanitizer_common/sanitizer_allocator.cc
index 15d01de86..db3ebb033 100644
--- a/lib/sanitizer_common/sanitizer_allocator.cc
+++ b/lib/sanitizer_common/sanitizer_allocator.cc
@@ -94,8 +94,7 @@ InternalAllocator *internal_allocator() {
     SpinMutexLock l(&internal_alloc_init_mu);
     if (atomic_load(&internal_allocator_initialized, memory_order_relaxed) ==
         0) {
-      internal_allocator_instance->Init(
-          /* may_return_null */ false, kReleaseToOSIntervalNever);
+      internal_allocator_instance->Init(kReleaseToOSIntervalNever);
       atomic_store(&internal_allocator_initialized, 1, memory_order_release);
     }
   }
@@ -162,7 +161,7 @@ void *InternalRealloc(void *addr, uptr size, InternalAllocatorCache *cache) {
 
 void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache) {
   if (CallocShouldReturnNullDueToOverflow(count, size))
-    return internal_allocator()->ReturnNullOrDieOnBadRequest();
+    return InternalAllocator::FailureHandler::OnBadRequest();
   void *p = InternalAlloc(count * size, cache);
   if (p) internal_memset(p, 0, count * size);
   return p;
@@ -209,12 +208,15 @@ bool CallocShouldReturnNullDueToOverflow(uptr size, uptr n) {
   return (max / size) < n;
 }
 
-static atomic_uint8_t reporting_out_of_memory = {0};
+static atomic_uint8_t allocator_out_of_memory = {0};
+static atomic_uint8_t allocator_may_return_null = {0};
 
-bool IsReportingOOM() { return atomic_load_relaxed(&reporting_out_of_memory); }
+bool IsAllocatorOutOfMemory() {
+  return atomic_load_relaxed(&allocator_out_of_memory);
+}
 
-void NORETURN ReportAllocatorCannotReturnNull(bool out_of_memory) {
-  if (out_of_memory) atomic_store_relaxed(&reporting_out_of_memory, 1);
+// Prints error message and kills the program.
+void NORETURN ReportAllocatorCannotReturnNull() {
   Report("%s's allocator is terminating the process instead of returning 0\n",
          SanitizerToolName);
   Report("If you don't like this behavior set allocator_may_return_null=1\n");
@@ -222,4 +224,35 @@ void NORETURN ReportAllocatorCannotReturnNull(bool out_of_memory) {
   Die();
 }
 
+bool AllocatorMayReturnNull() {
+  return atomic_load(&allocator_may_return_null, memory_order_relaxed);
+}
+
+void SetAllocatorMayReturnNull(bool may_return_null) {
+  atomic_store(&allocator_may_return_null, may_return_null,
+               memory_order_relaxed);
+}
+
+void *ReturnNullOrDieOnFailure::OnBadRequest() {
+  if (AllocatorMayReturnNull())
+    return nullptr;
+  ReportAllocatorCannotReturnNull();
+}
+
+void *ReturnNullOrDieOnFailure::OnOOM() {
+  atomic_store_relaxed(&allocator_out_of_memory, 1);
+  if (AllocatorMayReturnNull())
+    return nullptr;
+  ReportAllocatorCannotReturnNull();
+}
+
+void *DieOnFailure::OnBadRequest() {
+  ReportAllocatorCannotReturnNull();
+}
+
+void *DieOnFailure::OnOOM() {
+  atomic_store_relaxed(&allocator_out_of_memory, 1);
+  ReportAllocatorCannotReturnNull();
+}
+
 } // namespace __sanitizer