[sanitizer] Introduce a vDSO aware timing function

Summary:
See D40657 & D40679 for previous versions of this patch & description.

A couple of things were fixed here to have it not break some bots.
Weak symbols can't be used with `SANITIZER_GO` so the previous version was
breakin TsanGo. I set up some additional local tests and those pass now.

I changed the workaround for the glibc vDSO issue: `__progname` is initialized
after the vDSO and is actually public and of known type, unlike
`__vdso_clock_gettime`. This works better, and with all compilers.

The rest is the same.

Reviewers: alekseyshl

Reviewed By: alekseyshl

Subscribers: srhines, kubamracek, krytarowski, llvm-commits, #sanitizers

Differential Revision: https://reviews.llvm.org/D41121

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@320594 91177308-0d34-0410-b5e6-96231b3b80d8

12 files changed, 72 insertions, 7 deletions

diff --git a/lib/sanitizer_common/sanitizer_allocator_primary64.h b/lib/sanitizer_common/sanitizer_allocator_primary64.h
index e3d1bedff..651a64b04 100644
--- a/lib/sanitizer_common/sanitizer_allocator_primary64.h
+++ b/lib/sanitizer_common/sanitizer_allocator_primary64.h
@@ -697,7 +697,7 @@ class SizeClassAllocator64 {
         // Do it only when the feature is turned on, to avoid a potentially
         // extraneous syscall.
         if (ReleaseToOSIntervalMs() >= 0)
-          region->rtoi.last_release_at_ns = NanoTime();
+          region->rtoi.last_release_at_ns = MonotonicNanoTime();
       }
       // Do the mmap for the user memory.
       const uptr user_map_size =
@@ -827,7 +827,7 @@ class SizeClassAllocator64 {
         return;
 
       if (region->rtoi.last_release_at_ns + interval_ms * 1000000ULL >
-          NanoTime()) {
+          MonotonicNanoTime()) {
         return;  // Memory was returned recently.
       }
     }
@@ -844,6 +844,6 @@ class SizeClassAllocator64 {
       region->rtoi.num_releases += memory_mapper.GetReleasedRangesCount();
       region->rtoi.last_released_bytes = memory_mapper.GetReleasedBytes();
     }
-    region->rtoi.last_release_at_ns = NanoTime();
+    region->rtoi.last_release_at_ns = MonotonicNanoTime();
   }
 };
diff --git a/lib/sanitizer_common/sanitizer_common.h b/lib/sanitizer_common/sanitizer_common.h
index ceec58cd5..1fbaee7e3 100644
--- a/lib/sanitizer_common/sanitizer_common.h
+++ b/lib/sanitizer_common/sanitizer_common.h
@@ -295,6 +295,7 @@ uptr GetTlsSize();
 void SleepForSeconds(int seconds);
 void SleepForMillis(int millis);
 u64 NanoTime();
+u64 MonotonicNanoTime();
 int Atexit(void (*function)(void));
 void SortArray(uptr *array, uptr size);
 void SortArray(u32 *array, uptr size);
diff --git a/lib/sanitizer_common/sanitizer_common_interceptors.inc b/lib/sanitizer_common/sanitizer_common_interceptors.inc
index df9253bc5..720443d32 100644
--- a/lib/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/lib/sanitizer_common/sanitizer_common_interceptors.inc
@@ -2045,6 +2045,13 @@ INTERCEPTOR(int, clock_gettime, u32 clk_id, void *tp) {
   }
   return res;
 }
+namespace __sanitizer {
+extern "C" {
+int real_clock_gettime(u32 clk_id, void *tp) {
+  return REAL(clock_gettime)(clk_id, tp);
+}
+}  // extern "C"
+}  // namespace __sanitizer
 INTERCEPTOR(int, clock_settime, u32 clk_id, const void *tp) {
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, clock_settime, clk_id, tp);
diff --git a/lib/sanitizer_common/sanitizer_fuchsia.cc b/lib/sanitizer_common/sanitizer_fuchsia.cc
index 511d55e1d..936ec794b 100644
--- a/lib/sanitizer_common/sanitizer_fuchsia.cc
+++ b/lib/sanitizer_common/sanitizer_fuchsia.cc
@@ -51,6 +51,8 @@ unsigned int internal_sleep(unsigned int seconds) {
 
 u64 NanoTime() { return _zx_time_get(ZX_CLOCK_UTC); }
 
+u64 MonotonicNanoTime() { return _zx_time_get(ZX_CLOCK_MONOTONIC); }
+
 uptr internal_getpid() {
   zx_info_handle_basic_t info;
   zx_status_t status =
diff --git a/lib/sanitizer_common/sanitizer_linux.cc b/lib/sanitizer_common/sanitizer_linux.cc
index 6274b248e..89c76422c 100644
--- a/lib/sanitizer_common/sanitizer_linux.cc
+++ b/lib/sanitizer_common/sanitizer_linux.cc
@@ -459,6 +459,10 @@ u64 NanoTime() {
   return (u64)tv.tv_sec * 1000*1000*1000 + tv.tv_usec * 1000;
 }
 
+uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) {
+  return internal_syscall_ptr(SYSCALL(clock_gettime), clk_id, tp);
+}
+
 // Like getenv, but reads env directly from /proc (on Linux) or parses the
 // 'environ' array (on FreeBSD) and does not use libc. This function should be
 // called first inside __asan_init.
diff --git a/lib/sanitizer_common/sanitizer_linux.h b/lib/sanitizer_common/sanitizer_linux.h
index 88cac33d8..c10dcf563 100644
--- a/lib/sanitizer_common/sanitizer_linux.h
+++ b/lib/sanitizer_common/sanitizer_linux.h
@@ -46,6 +46,7 @@ uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count);
 uptr internal_sigaltstack(const void* ss, void* oss);
 uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
     __sanitizer_sigset_t *oldset);
+uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp);
 
 // Linux-only syscalls.
 #if SANITIZER_LINUX
diff --git a/lib/sanitizer_common/sanitizer_linux_libcdep.cc b/lib/sanitizer_common/sanitizer_linux_libcdep.cc
index c68f2146d..80d38215d 100644
--- a/lib/sanitizer_common/sanitizer_linux_libcdep.cc
+++ b/lib/sanitizer_common/sanitizer_linux_libcdep.cc
@@ -707,6 +707,47 @@ void LogMessageOnPrintf(const char *str) {
 
 #endif  // SANITIZER_LINUX
 
+#if SANITIZER_LINUX && !SANITIZER_GO
+// glibc crashes when using clock_gettime from a preinit_array function as the
+// vDSO function pointers haven't been initialized yet. __progname is
+// initialized after the vDSO function pointers, so if it exists, is not null
+// and is not empty, we can use clock_gettime.
+extern "C" SANITIZER_WEAK_ATTRIBUTE char *__progname;
+INLINE bool CanUseVDSO() {
+  // Bionic is safe, it checks for the vDSO function pointers to be initialized.
+  if (SANITIZER_ANDROID)
+    return true;
+  if (&__progname && __progname && *__progname)
+    return true;
+  return false;
+}
+
+// MonotonicNanoTime is a timing function that can leverage the vDSO by calling
+// clock_gettime. real_clock_gettime only exists if clock_gettime is
+// intercepted, so define it weakly and use it if available.
+extern "C" SANITIZER_WEAK_ATTRIBUTE
+int real_clock_gettime(u32 clk_id, void *tp);
+u64 MonotonicNanoTime() {
+  timespec ts;
+  if (CanUseVDSO()) {
+    if (&real_clock_gettime)
+      real_clock_gettime(CLOCK_MONOTONIC, &ts);
+    else
+      clock_gettime(CLOCK_MONOTONIC, &ts);
+  } else {
+    internal_clock_gettime(CLOCK_MONOTONIC, &ts);
+  }
+  return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;
+}
+#else
+// Non-Linux & Go always use the syscall.
+u64 MonotonicNanoTime() {
+  timespec ts;
+  internal_clock_gettime(CLOCK_MONOTONIC, &ts);
+  return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;
+}
+#endif  // SANITIZER_LINUX && !SANITIZER_GO
+
 } // namespace __sanitizer
 
-#endif // SANITIZER_FREEBSD || SANITIZER_LINUX
+#endif  // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
diff --git a/lib/sanitizer_common/sanitizer_mac.cc b/lib/sanitizer_common/sanitizer_mac.cc
index d28c2b10f..e1c51f580 100644
--- a/lib/sanitizer_common/sanitizer_mac.cc
+++ b/lib/sanitizer_common/sanitizer_mac.cc
@@ -365,6 +365,10 @@ u64 NanoTime() {
   return 0;
 }
 
+u64 MonotonicNanoTime() {
+  return 0;
+}
+
 uptr GetTlsSize() {
   return 0;
 }
diff --git a/lib/sanitizer_common/sanitizer_syscall_generic.inc b/lib/sanitizer_common/sanitizer_syscall_generic.inc
index 138d5ca90..ccc6dc360 100644
--- a/lib/sanitizer_common/sanitizer_syscall_generic.inc
+++ b/lib/sanitizer_common/sanitizer_syscall_generic.inc
@@ -36,6 +36,7 @@
 # define SYS_sigaltstack SYS___sigaltstack14
 # define SYS_sigprocmask SYS___sigprocmask14
 # define SYS_nanosleep SYS___nanosleep50
+# define SYS_clock_gettime SYS___clock_gettime50
 # if SANITIZER_WORDSIZE == 64
 #  define internal_syscall_ptr  __syscall
 # else
diff --git a/lib/sanitizer_common/sanitizer_win.cc b/lib/sanitizer_common/sanitizer_win.cc
index 4f82025ed..34bf1812d 100644
--- a/lib/sanitizer_common/sanitizer_win.cc
+++ b/lib/sanitizer_common/sanitizer_win.cc
@@ -505,6 +505,10 @@ u64 NanoTime() {
   return 0;
 }
 
+u64 MonotonicNanoTime() {
+  return 0;
+}
+
 void Abort() {
   internal__exit(3);
 }
diff --git a/lib/scudo/scudo_allocator.cpp b/lib/scudo/scudo_allocator.cpp
index d7df317ff..47804c44c 100644
--- a/lib/scudo/scudo_allocator.cpp
+++ b/lib/scudo/scudo_allocator.cpp
@@ -301,7 +301,7 @@ struct ScudoAllocator {
 
     CheckRssLimit = HardRssLimitMb || SoftRssLimitMb;
     if (CheckRssLimit)
-      atomic_store_relaxed(&RssLastCheckedAtNS, NanoTime());
+      atomic_store_relaxed(&RssLastCheckedAtNS, MonotonicNanoTime());
   }
 
   // Helper function that checks for a valid Scudo chunk. nullptr isn't.
@@ -319,7 +319,7 @@ struct ScudoAllocator {
   // it can, every 100ms, otherwise it will just return the current one.
   bool isRssLimitExceeded() {
     u64 LastCheck = atomic_load_relaxed(&RssLastCheckedAtNS);
-    const u64 CurrentCheck = NanoTime();
+    const u64 CurrentCheck = MonotonicNanoTime();
     if (LIKELY(CurrentCheck < LastCheck + (100ULL * 1000000ULL)))
       return atomic_load_relaxed(&RssLimitExceeded);
     if (!atomic_compare_exchange_weak(&RssLastCheckedAtNS, &LastCheck,
diff --git a/lib/scudo/scudo_tsd.h b/lib/scudo/scudo_tsd.h
index e8ba2cab7..80464b5ea 100644
--- a/lib/scudo/scudo_tsd.h
+++ b/lib/scudo/scudo_tsd.h
@@ -36,7 +36,7 @@ struct ALIGNED(64) ScudoTSD {
       return true;
     }
     if (atomic_load_relaxed(&Precedence) == 0)
-      atomic_store_relaxed(&Precedence, NanoTime());
+      atomic_store_relaxed(&Precedence, MonotonicNanoTime());
     return false;
   }