[scudo] Adding a public Scudo interface

Summary:
The first and only function to start with allows to set the soft or hard RSS
limit at runtime. Add associated tests.

Reviewers: alekseyshl

Reviewed By: alekseyshl

Subscribers: mgorny, #sanitizers, llvm-commits

Differential Revision: https://reviews.llvm.org/D41128

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@320611 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 42 insertions, 3 deletions

diff --git a/test/scudo/interface.cpp b/test/scudo/interface.cpp
index 16523bfe3..9d4433cc6 100644
--- a/test/scudo/interface.cpp
+++ b/test/scudo/interface.cpp
@@ -1,17 +1,21 @@
 // RUN: %clangxx_scudo %s -lstdc++ -o %t
-// RUN: %run %t ownership          2>&1
-// RUN: %run %t ownership-and-size 2>&1
-// RUN: %run %t heap-size          2>&1
+// RUN:                                                   %run %t ownership          2>&1
+// RUN:                                                   %run %t ownership-and-size 2>&1
+// RUN:                                                   %run %t heap-size          2>&1
+// RUN: %env_scudo_opts="allocator_may_return_null=1"     %run %t soft-limit         2>&1
+// RUN: %env_scudo_opts="allocator_may_return_null=1" not %run %t hard-limit         2>&1
 
 // Tests that the sanitizer interface functions behave appropriately.
 
 #include <stdlib.h>
 #include <assert.h>
 #include <string.h>
+#include <unistd.h>
 
 #include <vector>
 
 #include <sanitizer/allocator_interface.h>
+#include <sanitizer/scudo_interface.h>
 
 int main(int argc, char **argv)
 {
@@ -42,6 +46,41 @@ int main(int argc, char **argv)
     // allocator function.
     assert(__sanitizer_get_heap_size() >= 0);
   }
+  if (!strcmp(argv[1], "soft-limit")) {
+    // Verifies that setting the soft RSS limit at runtime works as expected.
+    std::vector<void *> pointers;
+    size_t size = 1 << 19;  // 512Kb
+    for (int i = 0; i < 5; i++)
+      pointers.push_back(malloc(size));
+    // Set the soft RSS limit to 1Mb.
+    __scudo_set_rss_limit(1, 0);
+    usleep(20000);
+    // The following allocation should return NULL.
+    void *p = malloc(size);
+    assert(!p);
+    // Remove the soft RSS limit.
+    __scudo_set_rss_limit(0, 0);
+    // The following allocation should succeed.
+    p = malloc(size);
+    assert(p);
+    free(p);
+    while (!pointers.empty()) {
+      free(pointers.back());
+      pointers.pop_back();
+    }
+  }
+  if (!strcmp(argv[1], "hard-limit")) {
+    // Verifies that setting the hard RSS limit at runtime works as expected.
+    std::vector<void *> pointers;
+    size_t size = 1 << 19;  // 512Kb
+    for (int i = 0; i < 5; i++)
+      pointers.push_back(malloc(size));
+    // Set the hard RSS limit to 1Mb
+    __scudo_set_rss_limit(1, 1);
+    usleep(20000);
+    // The following should trigger our death.
+    void *p = malloc(size);
+  }
 
   return 0;
 }