summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorKostya Kortchinsky <kostyak@google.com>2017-12-11 19:23:12 +0000
committerKostya Kortchinsky <kostyak@google.com>2017-12-11 19:23:12 +0000
commit313d4281293b9be28bc0a6af624c4729e4faae21 (patch)
tree09b12b80ba7450cd7208acc6a7e912c82d5035a9 /lib
parentb3c6c0c00d983413de921cbf5a139f0c6edcc72c (diff)
[sanitizer] Introduce a vDSO aware time function, and use it in the allocator [redo]
Summary: Redo of D40657, which had the initial discussion. The initial code had to move into a libcdep file, and things had to be shuffled accordingly. `NanoTime` is a time sink when checking whether or not to release memory to the OS. While reducing the amount of calls to said function is in the works, another solution that was found to be beneficial was to use a timing function that can leverage the vDSO. We hit a couple of snags along the way, like the fact that the glibc crashes when clock_gettime is called from a preinit_array, or the fact that `__vdso_clock_gettime` is mangled (for security purposes) and can't be used directly, and also that clock_gettime can be intercepted. The proposed solution takes care of all this as far as I can tell, and significantly improve performances and some Scudo load tests with memory reclaiming enabled. @mcgrathr: please feel free to follow up on https://reviews.llvm.org/D40657#940857 here. I posted a reply at https://reviews.llvm.org/D40657#940974. Reviewers: alekseyshl, krytarowski, flowerhack, mcgrathr, kubamracek Reviewed By: alekseyshl, krytarowski Subscribers: #sanitizers, mcgrathr, srhines, llvm-commits, kubamracek Differential Revision: https://reviews.llvm.org/D40679 git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@320409 91177308-0d34-0410-b5e6-96231b3b80d8
Diffstat (limited to 'lib')
-rw-r--r--lib/sanitizer_common/sanitizer_allocator_primary64.h6
-rw-r--r--lib/sanitizer_common/sanitizer_common.h1
-rw-r--r--lib/sanitizer_common/sanitizer_common_interceptors.inc7
-rw-r--r--lib/sanitizer_common/sanitizer_fuchsia.cc2
-rw-r--r--lib/sanitizer_common/sanitizer_linux.cc4
-rw-r--r--lib/sanitizer_common/sanitizer_linux.h1
-rw-r--r--lib/sanitizer_common/sanitizer_linux_libcdep.cc29
-rw-r--r--lib/sanitizer_common/sanitizer_mac.cc4
-rw-r--r--lib/sanitizer_common/sanitizer_syscall_generic.inc1
-rw-r--r--lib/sanitizer_common/sanitizer_win.cc4
-rw-r--r--lib/scudo/scudo_allocator.cpp4
-rw-r--r--lib/scudo/scudo_tsd.h2
12 files changed, 59 insertions, 6 deletions
diff --git a/lib/sanitizer_common/sanitizer_allocator_primary64.h b/lib/sanitizer_common/sanitizer_allocator_primary64.h
index e3d1bedff..651a64b04 100644
--- a/lib/sanitizer_common/sanitizer_allocator_primary64.h
+++ b/lib/sanitizer_common/sanitizer_allocator_primary64.h
@@ -697,7 +697,7 @@ class SizeClassAllocator64 {
// Do it only when the feature is turned on, to avoid a potentially
// extraneous syscall.
if (ReleaseToOSIntervalMs() >= 0)
- region->rtoi.last_release_at_ns = NanoTime();
+ region->rtoi.last_release_at_ns = MonotonicNanoTime();
}
// Do the mmap for the user memory.
const uptr user_map_size =
@@ -827,7 +827,7 @@ class SizeClassAllocator64 {
return;
if (region->rtoi.last_release_at_ns + interval_ms * 1000000ULL >
- NanoTime()) {
+ MonotonicNanoTime()) {
return; // Memory was returned recently.
}
}
@@ -844,6 +844,6 @@ class SizeClassAllocator64 {
region->rtoi.num_releases += memory_mapper.GetReleasedRangesCount();
region->rtoi.last_released_bytes = memory_mapper.GetReleasedBytes();
}
- region->rtoi.last_release_at_ns = NanoTime();
+ region->rtoi.last_release_at_ns = MonotonicNanoTime();
}
};
diff --git a/lib/sanitizer_common/sanitizer_common.h b/lib/sanitizer_common/sanitizer_common.h
index ceec58cd5..1fbaee7e3 100644
--- a/lib/sanitizer_common/sanitizer_common.h
+++ b/lib/sanitizer_common/sanitizer_common.h
@@ -295,6 +295,7 @@ uptr GetTlsSize();
void SleepForSeconds(int seconds);
void SleepForMillis(int millis);
u64 NanoTime();
+u64 MonotonicNanoTime();
int Atexit(void (*function)(void));
void SortArray(uptr *array, uptr size);
void SortArray(u32 *array, uptr size);
diff --git a/lib/sanitizer_common/sanitizer_common_interceptors.inc b/lib/sanitizer_common/sanitizer_common_interceptors.inc
index df9253bc5..720443d32 100644
--- a/lib/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/lib/sanitizer_common/sanitizer_common_interceptors.inc
@@ -2045,6 +2045,13 @@ INTERCEPTOR(int, clock_gettime, u32 clk_id, void *tp) {
}
return res;
}
+namespace __sanitizer {
+extern "C" {
+int real_clock_gettime(u32 clk_id, void *tp) {
+ return REAL(clock_gettime)(clk_id, tp);
+}
+} // extern "C"
+} // namespace __sanitizer
INTERCEPTOR(int, clock_settime, u32 clk_id, const void *tp) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, clock_settime, clk_id, tp);
diff --git a/lib/sanitizer_common/sanitizer_fuchsia.cc b/lib/sanitizer_common/sanitizer_fuchsia.cc
index 511d55e1d..936ec794b 100644
--- a/lib/sanitizer_common/sanitizer_fuchsia.cc
+++ b/lib/sanitizer_common/sanitizer_fuchsia.cc
@@ -51,6 +51,8 @@ unsigned int internal_sleep(unsigned int seconds) {
u64 NanoTime() { return _zx_time_get(ZX_CLOCK_UTC); }
+u64 MonotonicNanoTime() { return _zx_time_get(ZX_CLOCK_MONOTONIC); }
+
uptr internal_getpid() {
zx_info_handle_basic_t info;
zx_status_t status =
diff --git a/lib/sanitizer_common/sanitizer_linux.cc b/lib/sanitizer_common/sanitizer_linux.cc
index 6274b248e..89c76422c 100644
--- a/lib/sanitizer_common/sanitizer_linux.cc
+++ b/lib/sanitizer_common/sanitizer_linux.cc
@@ -459,6 +459,10 @@ u64 NanoTime() {
return (u64)tv.tv_sec * 1000*1000*1000 + tv.tv_usec * 1000;
}
+uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) {
+ return internal_syscall_ptr(SYSCALL(clock_gettime), clk_id, tp);
+}
+
// Like getenv, but reads env directly from /proc (on Linux) or parses the
// 'environ' array (on FreeBSD) and does not use libc. This function should be
// called first inside __asan_init.
diff --git a/lib/sanitizer_common/sanitizer_linux.h b/lib/sanitizer_common/sanitizer_linux.h
index 88cac33d8..c10dcf563 100644
--- a/lib/sanitizer_common/sanitizer_linux.h
+++ b/lib/sanitizer_common/sanitizer_linux.h
@@ -46,6 +46,7 @@ uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count);
uptr internal_sigaltstack(const void* ss, void* oss);
uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
__sanitizer_sigset_t *oldset);
+uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp);
// Linux-only syscalls.
#if SANITIZER_LINUX
diff --git a/lib/sanitizer_common/sanitizer_linux_libcdep.cc b/lib/sanitizer_common/sanitizer_linux_libcdep.cc
index c68f2146d..591cd87d2 100644
--- a/lib/sanitizer_common/sanitizer_linux_libcdep.cc
+++ b/lib/sanitizer_common/sanitizer_linux_libcdep.cc
@@ -707,6 +707,35 @@ void LogMessageOnPrintf(const char *str) {
#endif // SANITIZER_LINUX
+// glibc cannot use clock_gettime from a preinit_array function as the vDSO
+// function pointers haven't been initialized yet. To prevent a crash, we check
+// for the presence of the glibc symbol __vdso_clock_gettime, and verify that it
+// is not null (it can be mangled so we can't use it directly). Bionic's
+// clock_gettime actually falls back to the syscall in the same situation.
+extern "C" SANITIZER_WEAK_ATTRIBUTE void *__vdso_clock_gettime;
+bool CanUseLibcClockGetTime() {
+ return !SANITIZER_FREEBSD && !SANITIZER_NETBSD &&
+ (SANITIZER_ANDROID || (&__vdso_clock_gettime && __vdso_clock_gettime));
+}
+
+// MonotonicNanoTime is a timing function that can leverage the vDSO by calling
+// clock_gettime. real_clock_gettime only exists if clock_gettime is
+// intercepted, so define it weakly and use it if available.
+extern "C" SANITIZER_WEAK_ATTRIBUTE
+int real_clock_gettime(u32 clk_id, void *tp);
+u64 MonotonicNanoTime() {
+ timespec ts;
+ if (CanUseLibcClockGetTime()) {
+ if (&real_clock_gettime)
+ real_clock_gettime(CLOCK_MONOTONIC, &ts);
+ else
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ } else {
+ internal_clock_gettime(CLOCK_MONOTONIC, &ts);
+ }
+ return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;
+}
+
} // namespace __sanitizer
#endif // SANITIZER_FREEBSD || SANITIZER_LINUX
diff --git a/lib/sanitizer_common/sanitizer_mac.cc b/lib/sanitizer_common/sanitizer_mac.cc
index d28c2b10f..e1c51f580 100644
--- a/lib/sanitizer_common/sanitizer_mac.cc
+++ b/lib/sanitizer_common/sanitizer_mac.cc
@@ -365,6 +365,10 @@ u64 NanoTime() {
return 0;
}
+u64 MonotonicNanoTime() {
+ return 0;
+}
+
uptr GetTlsSize() {
return 0;
}
diff --git a/lib/sanitizer_common/sanitizer_syscall_generic.inc b/lib/sanitizer_common/sanitizer_syscall_generic.inc
index 138d5ca90..ccc6dc360 100644
--- a/lib/sanitizer_common/sanitizer_syscall_generic.inc
+++ b/lib/sanitizer_common/sanitizer_syscall_generic.inc
@@ -36,6 +36,7 @@
# define SYS_sigaltstack SYS___sigaltstack14
# define SYS_sigprocmask SYS___sigprocmask14
# define SYS_nanosleep SYS___nanosleep50
+# define SYS_clock_gettime SYS___clock_gettime50
# if SANITIZER_WORDSIZE == 64
# define internal_syscall_ptr __syscall
# else
diff --git a/lib/sanitizer_common/sanitizer_win.cc b/lib/sanitizer_common/sanitizer_win.cc
index 4f82025ed..34bf1812d 100644
--- a/lib/sanitizer_common/sanitizer_win.cc
+++ b/lib/sanitizer_common/sanitizer_win.cc
@@ -505,6 +505,10 @@ u64 NanoTime() {
return 0;
}
+u64 MonotonicNanoTime() {
+ return 0;
+}
+
void Abort() {
internal__exit(3);
}
diff --git a/lib/scudo/scudo_allocator.cpp b/lib/scudo/scudo_allocator.cpp
index 6383c6819..847a244c6 100644
--- a/lib/scudo/scudo_allocator.cpp
+++ b/lib/scudo/scudo_allocator.cpp
@@ -301,7 +301,7 @@ struct ScudoAllocator {
CheckRssLimit = HardRssLimitMb || SoftRssLimitMb;
if (CheckRssLimit)
- atomic_store_relaxed(&RssLastCheckedAtNS, NanoTime());
+ atomic_store_relaxed(&RssLastCheckedAtNS, MonotonicNanoTime());
}
// Helper function that checks for a valid Scudo chunk. nullptr isn't.
@@ -319,7 +319,7 @@ struct ScudoAllocator {
// it can, every 100ms, otherwise it will just return the current one.
bool isRssLimitExceeded() {
u64 LastCheck = atomic_load_relaxed(&RssLastCheckedAtNS);
- const u64 CurrentCheck = NanoTime();
+ const u64 CurrentCheck = MonotonicNanoTime();
if (LIKELY(CurrentCheck < LastCheck + (100ULL * 1000000ULL)))
return atomic_load_relaxed(&RssLimitExceeded);
if (!atomic_compare_exchange_weak(&RssLastCheckedAtNS, &LastCheck,
diff --git a/lib/scudo/scudo_tsd.h b/lib/scudo/scudo_tsd.h
index e8ba2cab7..80464b5ea 100644
--- a/lib/scudo/scudo_tsd.h
+++ b/lib/scudo/scudo_tsd.h
@@ -36,7 +36,7 @@ struct ALIGNED(64) ScudoTSD {
return true;
}
if (atomic_load_relaxed(&Precedence) == 0)
- atomic_store_relaxed(&Precedence, NanoTime());
+ atomic_store_relaxed(&Precedence, MonotonicNanoTime());
return false;
}