[Sanitizers] Operator new() interceptors always die on allocation error

Summary:
Operator new interceptors behavior is now controlled by their nothrow
property as well as by allocator_may_return_null flag value:

- allocator_may_return_null=* + new()        - die on allocation error
- allocator_may_return_null=0 + new(nothrow) - die on allocation error
- allocator_may_return_null=1 + new(nothrow) - return null

Ideally new() should throw std::bad_alloc exception, but that is not
trivial to achieve, hence TODO.

Reviewers: eugenis

Subscribers: kubamracek, llvm-commits

Differential Revision: https://reviews.llvm.org/D34731

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@306604 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 32 insertions, 13 deletions

diff --git a/test/scudo/sizes.cpp b/test/scudo/sizes.cpp
index 981b859a8..a0994c251 100644
--- a/test/scudo/sizes.cpp
+++ b/test/scudo/sizes.cpp
@@ -1,8 +1,12 @@
-// RUN: %clang_scudo %s -o %t
+// RUN: %clang_scudo %s -lstdc++ -o %t
 // RUN: SCUDO_OPTIONS=allocator_may_return_null=0 not %run %t malloc 2>&1 | FileCheck %s
 // RUN: SCUDO_OPTIONS=allocator_may_return_null=1     %run %t malloc 2>&1
 // RUN: SCUDO_OPTIONS=allocator_may_return_null=0 not %run %t calloc 2>&1 | FileCheck %s
 // RUN: SCUDO_OPTIONS=allocator_may_return_null=1     %run %t calloc 2>&1
+// RUN: SCUDO_OPTIONS=allocator_may_return_null=0 not %run %t new 2>&1 | FileCheck %s
+// RUN: SCUDO_OPTIONS=allocator_may_return_null=1 not %run %t new 2>&1 | FileCheck %s
+// RUN: SCUDO_OPTIONS=allocator_may_return_null=0 not %run %t new-nothrow 2>&1 | FileCheck %s
+// RUN: SCUDO_OPTIONS=allocator_may_return_null=1     %run %t new-nothrow 2>&1
 // RUN:                                               %run %t usable 2>&1
 
 // Tests for various edge cases related to sizes, notably the maximum size the
@@ -15,26 +19,38 @@
 #include <string.h>
 
 #include <limits>
+#include <new>
 
-int main(int argc, char **argv)
-{
+int main(int argc, char **argv) {
   assert(argc == 2);
-  if (!strcmp(argv[1], "malloc")) {
-    // Currently the maximum size the allocator can allocate is 1ULL<<40 bytes.
-    size_t size = std::numeric_limits<size_t>::max();
-    void *p = malloc(size);
+  const char *action = argv[1];
+  fprintf(stderr, "%s:\n", action);
+
+#if __LP64__ || defined(_WIN64)
+  static const size_t kMaxAllowedMallocSize = 1ULL << 40;
+  static const size_t kChunkHeaderSize = 16;
+#else
+  static const size_t kMaxAllowedMallocSize = 2UL << 30;
+  static const size_t kChunkHeaderSize = 8;
+#endif
+
+  if (!strcmp(action, "malloc")) {
+    void *p = malloc(kMaxAllowedMallocSize);
     assert(!p);
-    size = (1ULL << 40) - 16;
-    p = malloc(size);
+    p = malloc(kMaxAllowedMallocSize - kChunkHeaderSize);
     assert(!p);
-  }
-  if (!strcmp(argv[1], "calloc")) {
+  } else if (!strcmp(action, "calloc")) {
     // Trigger an overflow in calloc.
     size_t size = std::numeric_limits<size_t>::max();
     void *p = calloc((size / 0x1000) + 1, 0x1000);
     assert(!p);
-  }
-  if (!strcmp(argv[1], "usable")) {
+  } else if (!strcmp(action, "new")) {
+    void *p = operator new(kMaxAllowedMallocSize);
+    assert(!p);
+  } else if (!strcmp(action, "new-nothrow")) {
+    void *p = operator new(kMaxAllowedMallocSize, std::nothrow);
+    assert(!p);
+  } else if (!strcmp(action, "usable")) {
     // Playing with the actual usable size of a chunk.
     void *p = malloc(1007);
     assert(p);
@@ -47,7 +63,10 @@ int main(int argc, char **argv)
     assert(size >= 2014);
     memset(p, 'B', size);
     free(p);
+  } else {
+    assert(0);
   }
+
   return 0;
 }