[Sanitizers] Operator new() interceptors always die on allocation error

Summary:
Operator new interceptors behavior is now controlled by their nothrow
property as well as by allocator_may_return_null flag value:

- allocator_may_return_null=* + new()        - die on allocation error
- allocator_may_return_null=0 + new(nothrow) - die on allocation error
- allocator_may_return_null=1 + new(nothrow) - return null

Ideally new() should throw std::bad_alloc exception, but that is not
trivial to achieve, hence TODO.

Reviewers: eugenis

Subscribers: kubamracek, llvm-commits

Differential Revision: https://reviews.llvm.org/D34731

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@306604 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 83 insertions, 33 deletions

diff --git a/test/tsan/allocator_returns_null.cc b/test/tsan/allocator_returns_null.cc
index 66930076a..852dda035 100644
--- a/test/tsan/allocator_returns_null.cc
+++ b/test/tsan/allocator_returns_null.cc
@@ -1,56 +1,90 @@
-// Test the behavior of malloc/calloc/realloc when the allocation size is huge.
+// Test the behavior of malloc/calloc/realloc/new when the allocation size is
+// more than TSan allocator's max allowed one.
 // By default (allocator_may_return_null=0) the process should crash.
-// With allocator_may_return_null=1 the allocator should return 0.
+// With allocator_may_return_null=1 the allocator should return 0, except the
+// operator new(), which should crash anyway (operator new(std::nothrow) should
+// return nullptr, indeed).
 //
 // RUN: %clangxx_tsan -O0 %s -o %t
 // RUN: not %run %t malloc 2>&1 | FileCheck %s --check-prefix=CHECK-mCRASH
-// RUN: %env_tsan_opts=allocator_may_return_null=0 not %run %t malloc 2>&1 | FileCheck %s --check-prefix=CHECK-mCRASH
-// RUN: %env_tsan_opts=allocator_may_return_null=0 not %run %t calloc 2>&1 | FileCheck %s --check-prefix=CHECK-cCRASH
-// RUN: %env_tsan_opts=allocator_may_return_null=0 not %run %t calloc-overflow 2>&1 | FileCheck %s --check-prefix=CHECK-coCRASH
-// RUN: %env_tsan_opts=allocator_may_return_null=0 not %run %t realloc 2>&1 | FileCheck %s --check-prefix=CHECK-rCRASH
-// RUN: %env_tsan_opts=allocator_may_return_null=0 not %run %t realloc-after-malloc 2>&1 | FileCheck %s --check-prefix=CHECK-mrCRASH
+// RUN: %env_tsan_opts=allocator_may_return_null=0 not %run %t malloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-mCRASH
+// RUN: %env_tsan_opts=allocator_may_return_null=1     %run %t malloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-mNULL
+// RUN: %env_tsan_opts=allocator_may_return_null=0 not %run %t calloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-cCRASH
+// RUN: %env_tsan_opts=allocator_may_return_null=1     %run %t calloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-cNULL
+// RUN: %env_tsan_opts=allocator_may_return_null=0 not %run %t calloc-overflow 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-coCRASH
+// RUN: %env_tsan_opts=allocator_may_return_null=1     %run %t calloc-overflow 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-coNULL
+// RUN: %env_tsan_opts=allocator_may_return_null=0 not %run %t realloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-rCRASH
+// RUN: %env_tsan_opts=allocator_may_return_null=1     %run %t realloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-rNULL
+// RUN: %env_tsan_opts=allocator_may_return_null=0 not %run %t realloc-after-malloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-mrCRASH
+// RUN: %env_tsan_opts=allocator_may_return_null=1     %run %t realloc-after-malloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-mrNULL
+// RUN: %env_tsan_opts=allocator_may_return_null=0 not %run %t new 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-nCRASH
+// RUN: %env_tsan_opts=allocator_may_return_null=1 not %run %t new 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-nCRASH
+// RUN: %env_tsan_opts=allocator_may_return_null=0 not %run %t new-nothrow 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-nnCRASH
+// RUN: %env_tsan_opts=allocator_may_return_null=1     %run %t new-nothrow 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-nnNULL
 
-#include <limits.h>
-#include <stdlib.h>
+#include <assert.h>
 #include <string.h>
 #include <stdio.h>
-#include <assert.h>
+#include <stdlib.h>
 #include <limits>
+#include <new>
+
 int main(int argc, char **argv) {
-  volatile size_t size = std::numeric_limits<size_t>::max() - 10000;
+  // Disable stderr buffering. Needed on Windows.
+  setvbuf(stderr, NULL, _IONBF, 0);
+
   assert(argc == 2);
-  char *x = 0;
-  if (!strcmp(argv[1], "malloc")) {
-    fprintf(stderr, "malloc:\n");
-    x = (char*)malloc(size);
-  }
-  if (!strcmp(argv[1], "calloc")) {
-    fprintf(stderr, "calloc:\n");
-    x = (char*)calloc(size / 4, 4);
-  }
+  const char *action = argv[1];
+  fprintf(stderr, "%s:\n", action);
+
+  static const size_t kMaxAllowedMallocSizePlusOne = (1ULL << 40) + 1;
 
-  if (!strcmp(argv[1], "calloc-overflow")) {
-    fprintf(stderr, "calloc-overflow:\n");
+  void *x = 0;
+  if (!strcmp(action, "malloc")) {
+    x = malloc(kMaxAllowedMallocSizePlusOne);
+  } else if (!strcmp(action, "calloc")) {
+    x = calloc((kMaxAllowedMallocSizePlusOne / 4) + 1, 4);
+  } else if (!strcmp(action, "calloc-overflow")) {
     volatile size_t kMaxSizeT = std::numeric_limits<size_t>::max();
     size_t kArraySize = 4096;
     volatile size_t kArraySize2 = kMaxSizeT / kArraySize + 10;
-    x = (char*)calloc(kArraySize, kArraySize2);
-  }
-
-  if (!strcmp(argv[1], "realloc")) {
-    fprintf(stderr, "realloc:\n");
-    x = (char*)realloc(0, size);
-  }
-  if (!strcmp(argv[1], "realloc-after-malloc")) {
-    fprintf(stderr, "realloc-after-malloc:\n");
+    x = calloc(kArraySize, kArraySize2);
+  } else if (!strcmp(action, "realloc")) {
+    x = realloc(0, kMaxAllowedMallocSizePlusOne);
+  } else if (!strcmp(action, "realloc-after-malloc")) {
     char *t = (char*)malloc(100);
     *t = 42;
-    x = (char*)realloc(t, size);
+    x = realloc(t, kMaxAllowedMallocSizePlusOne);
     assert(*t == 42);
+  } else if (!strcmp(action, "new")) {
+    x = operator new(kMaxAllowedMallocSizePlusOne);
+  } else if (!strcmp(action, "new-nothrow")) {
+    x = operator new(kMaxAllowedMallocSizePlusOne, std::nothrow);
+  } else {
+    assert(0);
   }
-  fprintf(stderr, "x: %p\n", x);
+
+  // The NULL pointer is printed differently on different systems, while (long)0
+  // is always the same.
+  fprintf(stderr, "x: %lx\n", (long)x);
+  free(x);
   return x != 0;
 }
+
 // CHECK-mCRASH: malloc:
 // CHECK-mCRASH: ThreadSanitizer's allocator is terminating the process
 // CHECK-cCRASH: calloc:
@@ -61,4 +95,20 @@ int main(int argc, char **argv) {
 // CHECK-rCRASH: ThreadSanitizer's allocator is terminating the process
 // CHECK-mrCRASH: realloc-after-malloc:
 // CHECK-mrCRASH: ThreadSanitizer's allocator is terminating the process
+// CHECK-nCRASH: new:
+// CHECK-nCRASH: ThreadSanitizer's allocator is terminating the process
+// CHECK-nnCRASH: new-nothrow:
+// CHECK-nnCRASH: ThreadSanitizer's allocator is terminating the process
 
+// CHECK-mNULL: malloc:
+// CHECK-mNULL: x: 0
+// CHECK-cNULL: calloc:
+// CHECK-cNULL: x: 0
+// CHECK-coNULL: calloc-overflow:
+// CHECK-coNULL: x: 0
+// CHECK-rNULL: realloc:
+// CHECK-rNULL: x: 0
+// CHECK-mrNULL: realloc-after-malloc:
+// CHECK-mrNULL: x: 0
+// CHECK-nnNULL: new-nothrow:
+// CHECK-nnNULL: x: 0