[msan] Implement allocator_may_return_null=1 in MemorySanitizer.

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@192687 91177308-0d34-0410-b5e6-96231b3b80d8

2 files changed, 90 insertions, 2 deletions

diff --git a/lib/msan/lit_tests/allocator_returns_null.cc b/lib/msan/lit_tests/allocator_returns_null.cc
new file mode 100644
index 000000000..aaa85cce7
--- /dev/null
+++ b/lib/msan/lit_tests/allocator_returns_null.cc
@@ -0,0 +1,81 @@
+// Test the behavior of malloc/calloc/realloc when the allocation size is huge.
+// By default (allocator_may_return_null=0) the process should crash.
+// With allocator_may_return_null=1 the allocator should return 0.
+//
+// RUN: %clangxx_msan -O0 %s -o %t
+// RUN: not %t malloc 2>&1 | FileCheck %s --check-prefix=CHECK-mCRASH
+// RUN: MSAN_OPTIONS=allocator_may_return_null=0 not %t malloc 2>&1 | FileCheck %s --check-prefix=CHECK-mCRASH
+// RUN: MSAN_OPTIONS=allocator_may_return_null=1     %t malloc 2>&1 | FileCheck %s --check-prefix=CHECK-mNULL
+// RUN: MSAN_OPTIONS=allocator_may_return_null=0 not %t calloc 2>&1 | FileCheck %s --check-prefix=CHECK-cCRASH
+// RUN: MSAN_OPTIONS=allocator_may_return_null=1     %t calloc 2>&1 | FileCheck %s --check-prefix=CHECK-cNULL
+// RUN: MSAN_OPTIONS=allocator_may_return_null=0 not %t calloc-overflow 2>&1 | FileCheck %s --check-prefix=CHECK-coCRASH
+// RUN: MSAN_OPTIONS=allocator_may_return_null=1     %t calloc-overflow 2>&1 | FileCheck %s --check-prefix=CHECK-coNULL
+// RUN: MSAN_OPTIONS=allocator_may_return_null=0 not %t realloc 2>&1 | FileCheck %s --check-prefix=CHECK-rCRASH
+// RUN: MSAN_OPTIONS=allocator_may_return_null=1     %t realloc 2>&1 | FileCheck %s --check-prefix=CHECK-rNULL
+// RUN: MSAN_OPTIONS=allocator_may_return_null=0 not %t realloc-after-malloc 2>&1 | FileCheck %s --check-prefix=CHECK-mrCRASH
+// RUN: MSAN_OPTIONS=allocator_may_return_null=1     %t realloc-after-malloc 2>&1 | FileCheck %s --check-prefix=CHECK-mrNULL
+
+#include <limits.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <assert.h>
+#include <limits>
+int main(int argc, char **argv) {
+  volatile size_t size = std::numeric_limits<size_t>::max() - 10000;
+  assert(argc == 2);
+  char *x = 0;
+  if (!strcmp(argv[1], "malloc")) {
+    fprintf(stderr, "malloc:\n");
+    x = (char*)malloc(size);
+  }
+  if (!strcmp(argv[1], "calloc")) {
+    fprintf(stderr, "calloc:\n");
+    x = (char*)calloc(size / 4, 4);
+  }
+
+  if (!strcmp(argv[1], "calloc-overflow")) {
+    fprintf(stderr, "calloc-overflow:\n");
+    volatile size_t kMaxSizeT = std::numeric_limits<size_t>::max();
+    size_t kArraySize = 4096;
+    volatile size_t kArraySize2 = kMaxSizeT / kArraySize + 10;
+    x = (char*)calloc(kArraySize, kArraySize2);
+  }
+
+  if (!strcmp(argv[1], "realloc")) {
+    fprintf(stderr, "realloc:\n");
+    x = (char*)realloc(0, size);
+  }
+  if (!strcmp(argv[1], "realloc-after-malloc")) {
+    fprintf(stderr, "realloc-after-malloc:\n");
+    char *t = (char*)malloc(100);
+    *t = 42;
+    x = (char*)realloc(t, size);
+    assert(*t == 42);
+  }
+  // The NULL pointer is printed differently on different systems, while (long)0
+  // is always the same.
+  fprintf(stderr, "x: %lx\n", (long)x);
+  return x != 0;
+}
+// CHECK-mCRASH: malloc:
+// CHECK-mCRASH: MemorySanitizer's allocator is terminating the process
+// CHECK-cCRASH: calloc:
+// CHECK-cCRASH: MemorySanitizer's allocator is terminating the process
+// CHECK-coCRASH: calloc-overflow:
+// CHECK-coCRASH: MemorySanitizer's allocator is terminating the process
+// CHECK-rCRASH: realloc:
+// CHECK-rCRASH: MemorySanitizer's allocator is terminating the process
+// CHECK-mrCRASH: realloc-after-malloc:
+// CHECK-mrCRASH: MemorySanitizer's allocator is terminating the process
+
+// CHECK-mNULL: malloc:
+// CHECK-mNULL: x: 0
+// CHECK-cNULL: calloc:
+// CHECK-cNULL: x: 0
+// CHECK-coNULL: calloc-overflow:
+// CHECK-coNULL: x: 0
+// CHECK-rNULL: realloc:
+// CHECK-rNULL: x: 0
+// CHECK-mrNULL: realloc-after-malloc:
+// CHECK-mrNULL: x: 0
diff --git a/lib/msan/msan_allocator.cc b/lib/msan/msan_allocator.cc
index 0fe1297a8..04d6fedb2 100644
--- a/lib/msan/msan_allocator.cc
+++ b/lib/msan/msan_allocator.cc
@@ -25,6 +25,7 @@ struct Metadata {
 static const uptr kAllocatorSpace = 0x600000000000ULL;
 static const uptr kAllocatorSize   = 0x80000000000;  // 8T.
 static const uptr kMetadataSize  = sizeof(Metadata);
+static const uptr kMaxAllowedMallocSize = 8UL << 30;
 
 typedef SizeClassAllocator64<kAllocatorSpace, kAllocatorSize, kMetadataSize,
                              DefaultSizeClassMap> PrimaryAllocator;
@@ -48,6 +49,11 @@ static inline void Init() {
 static void *MsanAllocate(StackTrace *stack, uptr size,
                           uptr alignment, bool zeroise) {
   Init();
+  if (size > kMaxAllowedMallocSize) {
+    Report("WARNING: MemorySanitizer failed to allocate %p bytes\n",
+           (void *)size);
+    return AllocatorReturnNull();
+  }
   void *res = allocator.Allocate(&cache, size, alignment, false);
   Metadata *meta = reinterpret_cast<Metadata*>(allocator.GetMetaData(res));
   meta->requested_size = size;
@@ -110,9 +116,10 @@ void *MsanReallocate(StackTrace *stack, void *old_p, uptr new_size,
   uptr memcpy_size = Min(new_size, old_size);
   void *new_p = MsanAllocate(stack, new_size, alignment, zeroise);
   // Printf("realloc: old_size %zd new_size %zd\n", old_size, new_size);
-  if (new_p)
+  if (new_p) {
     __msan_memcpy(new_p, old_p, memcpy_size);
-  MsanDeallocate(stack, old_p);
+    MsanDeallocate(stack, old_p);
+  }
   return new_p;
 }