tsan: support mmap(MAP_32BIT)

Allow user memory in the first TB of address space.
This also enabled non-pie binaries and freebsd.
Fixes issue:
https://code.google.com/p/thread-sanitizer/issues/detail?id=5



git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@220571 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 197 insertions, 97 deletions

diff --git a/lib/tsan/rtl/tsan_platform.h b/lib/tsan/rtl/tsan_platform.h
index 35f4a9195..45f8631fd 100644
--- a/lib/tsan/rtl/tsan_platform.h
+++ b/lib/tsan/rtl/tsan_platform.h
@@ -12,124 +12,223 @@
 // Platform-specific code.
 //===----------------------------------------------------------------------===//
 
+#ifndef TSAN_PLATFORM_H
+#define TSAN_PLATFORM_H
+
+#if !defined(__LP64__) && !defined(_WIN64)
+# error "Only 64-bit is supported"
+#endif
+
+#include "tsan_defs.h"
+#include "tsan_trace.h"
+
+namespace __tsan {
+
+#if !defined(TSAN_GO)
+
 /*
-C++ linux memory layout:
-0000 0000 0000 - 03c0 0000 0000: protected
-03c0 0000 0000 - 1000 0000 0000: shadow
-1000 0000 0000 - 3000 0000 0000: protected
+C/C++ on linux and freebsd
+0000 0000 1000 - 0100 0000 0000: main binary and/or MAP_32BIT mappings
+0100 0000 0000 - 0200 0000 0000: -
+0200 0000 0000 - 1000 0000 0000: shadow
+1000 0000 0000 - 3000 0000 0000: -
 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
-4000 0000 0000 - 6000 0000 0000: protected
+4000 0000 0000 - 6000 0000 0000: -
 6000 0000 0000 - 6200 0000 0000: traces
 6200 0000 0000 - 7d00 0000 0000: -
 7d00 0000 0000 - 7e00 0000 0000: heap
-7e00 0000 0000 - 7fff ffff ffff: modules and main thread stack
+7e00 0000 0000 - 7e80 0000 0000: -
+7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
+*/
 
-Go linux and darwin memory layout:
-0000 0000 0000 - 0000 1000 0000: executable
-0000 1000 0000 - 00f8 0000 0000: -
+const uptr kMetaShadowBeg = 0x300000000000ull;
+const uptr kMetaShadowEnd = 0x400000000000ull;
+const uptr kTraceMemBeg   = 0x600000000000ull;
+const uptr kTraceMemEnd   = 0x620000000000ull;
+const uptr kShadowBeg     = 0x020000000000ull;
+const uptr kShadowEnd     = 0x100000000000ull;
+const uptr kHeapMemBeg    = 0x7d0000000000ull;
+const uptr kHeapMemEnd    = 0x7e0000000000ull;
+const uptr kLoAppMemBeg   = 0x000000001000ull;
+const uptr kLoAppMemEnd   = 0x010000000000ull;
+const uptr kHiAppMemBeg   = 0x7e8000000000ull;
+const uptr kHiAppMemEnd   = 0x800000000000ull;
+const uptr kAppMemMsk     = 0x7c0000000000ull;
+const uptr kAppMemXor     = 0x020000000000ull;
+
+ALWAYS_INLINE
+bool IsAppMem(uptr mem) {
+  return (mem >= kHeapMemBeg && mem < kHeapMemEnd) ||
+         (mem >= kLoAppMemBeg && mem < kLoAppMemEnd) ||
+         (mem >= kHiAppMemBeg && mem < kHiAppMemEnd);
+}
+
+ALWAYS_INLINE
+bool IsShadowMem(uptr mem) {
+  return mem >= kShadowBeg && mem <= kShadowEnd;
+}
+
+ALWAYS_INLINE
+bool IsMetaMem(uptr mem) {
+  return mem >= kMetaShadowBeg && mem <= kMetaShadowEnd;
+}
+
+ALWAYS_INLINE
+uptr MemToShadow(uptr x) {
+  DCHECK(IsAppMem(x));
+  return (((x) & ~(kAppMemMsk | (kShadowCell - 1)))
+      ^ kAppMemXor) * kShadowCnt;
+}
+
+ALWAYS_INLINE
+u32 *MemToMeta(uptr x) {
+  DCHECK(IsAppMem(x));
+  return (u32*)(((((x) & ~(kAppMemMsk | (kMetaShadowCell - 1)))
+      ^ kAppMemXor) / kMetaShadowCell * kMetaShadowSize) | kMetaShadowBeg);
+}
+
+ALWAYS_INLINE
+uptr ShadowToMem(uptr s) {
+  CHECK(IsShadowMem(s));
+  if (s >= MemToShadow(kLoAppMemBeg) && s <= MemToShadow(kLoAppMemEnd - 1))
+    return (s / kShadowCnt) ^ kAppMemXor;
+  else
+    return ((s / kShadowCnt) ^ kAppMemXor) | kAppMemMsk;
+}
+
+static USED uptr UserRegions[] = {
+  kLoAppMemBeg, kLoAppMemEnd,
+  kHiAppMemBeg, kHiAppMemEnd,
+  kHeapMemBeg,  kHeapMemEnd,
+};
+
+#elif defined(TSAN_GO) && !SANITIZER_WINDOWS
+
+/* Go on linux, darwin and freebsd
+0000 0000 1000 - 0000 1000 0000: executable
+0000 1000 0000 - 00c0 0000 0000: -
 00c0 0000 0000 - 00e0 0000 0000: heap
-00e0 0000 0000 - 1000 0000 0000: -
-1000 0000 0000 - 1380 0000 0000: shadow
-1460 0000 0000 - 2000 0000 0000: -
+00e0 0000 0000 - 2000 0000 0000: -
+2000 0000 0000 - 2380 0000 0000: shadow
+2380 0000 0000 - 3000 0000 0000: -
 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
 4000 0000 0000 - 6000 0000 0000: -
 6000 0000 0000 - 6200 0000 0000: traces
-6200 0000 0000 - 7fff ffff ffff: -
+6200 0000 0000 - 8000 0000 0000: -
+*/
+
+const uptr kMetaShadowBeg = 0x300000000000ull;
+const uptr kMetaShadowEnd = 0x400000000000ull;
+const uptr kTraceMemBeg   = 0x600000000000ull;
+const uptr kTraceMemEnd   = 0x620000000000ull;
+const uptr kShadowBeg     = 0x200000000000ull;
+const uptr kShadowEnd     = 0x238000000000ull;
+const uptr kAppMemBeg     = 0x000000001000ull;
+const uptr kAppMemEnd     = 0x00e000000000ull;
+
+ALWAYS_INLINE
+bool IsAppMem(uptr mem) {
+  return mem >= kAppMemBeg && mem < kAppMemEnd;
+}
+
+ALWAYS_INLINE
+bool IsShadowMem(uptr mem) {
+  return mem >= kShadowBeg && mem <= kShadowEnd;
+}
+
+ALWAYS_INLINE
+bool IsMetaMem(uptr mem) {
+  return mem >= kMetaShadowBeg && mem <= kMetaShadowEnd;
+}
+
+ALWAYS_INLINE
+uptr MemToShadow(uptr x) {
+  DCHECK(IsAppMem(x));
+  return ((x & ~(kShadowCell - 1)) * kShadowCnt) | kShadowBeg;
+}
 
-Go windows memory layout:
-0000 0000 0000 - 0000 1000 0000: executable
+ALWAYS_INLINE
+u32 *MemToMeta(uptr x) {
+  DCHECK(IsAppMem(x));
+  return (u32*)(((x & ~(kMetaShadowCell - 1)) / \
+      kMetaShadowCell * kMetaShadowSize) | kMetaShadowBeg);
+}
+
+ALWAYS_INLINE
+uptr ShadowToMem(uptr s) {
+  CHECK(IsShadowMem(s));
+  return (s & ~kShadowBeg) / kShadowCnt;
+}
+
+static USED uptr UserRegions[] = {
+  kAppMemBeg, kAppMemEnd,
+};
+
+#elif defined(TSAN_GO) && SANITIZER_WINDOWS
+
+/* Go on windows
+0000 0000 1000 - 0000 1000 0000: executable
 0000 1000 0000 - 00f8 0000 0000: -
 00c0 0000 0000 - 00e0 0000 0000: heap
 00e0 0000 0000 - 0100 0000 0000: -
-0100 0000 0000 - 0560 0000 0000: shadow
+0100 0000 0000 - 0380 0000 0000: shadow
+0380 0000 0000 - 0560 0000 0000: -
 0560 0000 0000 - 0760 0000 0000: traces
 0760 0000 0000 - 07d0 0000 0000: metainfo (memory blocks and sync objects)
-07d0 0000 0000 - 07ff ffff ffff: -
+07d0 0000 0000 - 8000 0000 0000: -
 */
 
-#ifndef TSAN_PLATFORM_H
-#define TSAN_PLATFORM_H
-
-#include "tsan_defs.h"
-#include "tsan_trace.h"
-
-#if defined(__LP64__) || defined(_WIN64)
-namespace __tsan {
-
-#if defined(TSAN_GO)
-static const uptr kLinuxAppMemBeg = 0x000000000000ULL;
-static const uptr kLinuxAppMemEnd = 0x04dfffffffffULL;
-# if SANITIZER_WINDOWS
-static const uptr kLinuxShadowMsk = 0x010000000000ULL;
-static const uptr kMetaShadow     = 0x076000000000ULL;
-static const uptr kMetaSize       = 0x007000000000ULL;
-# else  // if SANITIZER_WINDOWS
-static const uptr kLinuxShadowMsk = 0x200000000000ULL;
-static const uptr kMetaShadow     = 0x300000000000ULL;
-static const uptr kMetaSize       = 0x100000000000ULL;
-# endif  // if SANITIZER_WINDOWS
-#else  // defined(TSAN_GO)
-static const uptr kMetaShadow     = 0x300000000000ULL;
-static const uptr kMetaSize       = 0x100000000000ULL;
-static const uptr kLinuxAppMemBeg = 0x7cf000000000ULL;
-static const uptr kLinuxAppMemEnd = 0x7fffffffffffULL;
-#endif
+const uptr kMetaShadowBeg = 0x076000000000ull;
+const uptr kMetaShadowEnd = 0x07d000000000ull;
+const uptr kTraceMemBeg   = 0x056000000000ull;
+const uptr kTraceMemEnd   = 0x076000000000ull;
+const uptr kShadowBeg     = 0x010000000000ull;
+const uptr kShadowEnd     = 0x038000000000ull;
+const uptr kAppMemBeg     = 0x000000001000ull;
+const uptr kAppMemEnd     = 0x00e000000000ull;
 
-static const uptr kLinuxAppMemMsk = 0x7c0000000000ULL;
+ALWAYS_INLINE
+bool IsAppMem(uptr mem) {
+  return mem >= kAppMemBeg && mem < kAppMemEnd;
+}
 
-#if SANITIZER_WINDOWS
-const uptr kTraceMemBegin = 0x056000000000ULL;
-#else
-const uptr kTraceMemBegin = 0x600000000000ULL;
-#endif
-const uptr kTraceMemSize = 0x020000000000ULL;
-
-// This has to be a macro to allow constant initialization of constants below.
-#ifndef TSAN_GO
-#define MemToShadow(addr) \
-    ((((uptr)addr) & ~(kLinuxAppMemMsk | (kShadowCell - 1))) * kShadowCnt)
-#define MemToMeta(addr) \
-    (u32*)(((((uptr)addr) & ~(kLinuxAppMemMsk | (kMetaShadowCell - 1))) \
-    / kMetaShadowCell * kMetaShadowSize) | kMetaShadow)
-#else
-#define MemToShadow(addr) \
-    (((((uptr)addr) & ~(kShadowCell - 1)) * kShadowCnt) | kLinuxShadowMsk)
-#define MemToMeta(addr) \
-    (u32*)(((((uptr)addr) & ~(kMetaShadowCell - 1)) \
-    / kMetaShadowCell * kMetaShadowSize) | kMetaShadow)
-#endif
+ALWAYS_INLINE
+bool IsShadowMem(uptr mem) {
+  return mem >= kShadowBeg && mem <= kShadowEnd;
+}
 
-static const uptr kLinuxShadowBeg = MemToShadow(kLinuxAppMemBeg);
-static const uptr kLinuxShadowEnd =
-    MemToShadow(kLinuxAppMemEnd) | 0xff;
+ALWAYS_INLINE
+bool IsMetaMem(uptr mem) {
+  return mem >= kMetaShadowBeg && mem <= kMetaShadowEnd;
+}
 
-static inline bool IsAppMem(uptr mem) {
-#if defined(TSAN_GO)
-  return mem <= kLinuxAppMemEnd;
-#else
-  return mem >= kLinuxAppMemBeg && mem <= kLinuxAppMemEnd;
-#endif
+ALWAYS_INLINE
+uptr MemToShadow(uptr x) {
+  DCHECK(IsAppMem(x));
+  return ((x & ~(kShadowCell - 1)) * kShadowCnt) | kShadowBeg;
 }
 
-static inline bool IsShadowMem(uptr mem) {
-  return mem >= kLinuxShadowBeg && mem <= kLinuxShadowEnd;
+ALWAYS_INLINE
+u32 *MemToMeta(uptr x) {
+  DCHECK(IsAppMem(x));
+  return (u32*)(((x & ~(kMetaShadowCell - 1)) / \
+      kMetaShadowCell * kMetaShadowSize) | kMetaShadowEnd);
 }
 
-static inline uptr ShadowToMem(uptr shadow) {
-  CHECK(IsShadowMem(shadow));
-#ifdef TSAN_GO
-  return (shadow & ~kLinuxShadowMsk) / kShadowCnt;
-#else
-  return (shadow / kShadowCnt) | kLinuxAppMemMsk;
-#endif
+ALWAYS_INLINE
+uptr ShadowToMem(uptr s) {
+  CHECK(IsShadowMem(s));
+  // FIXME(dvyukov): this is most likely wrong as the mapping is not bijection.
+  return (x & ~kShadowBeg) / kShadowCnt;
 }
 
-void FlushShadowMemory();
-void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive);
-uptr GetRSS();
+static USED uptr UserRegions[] = {
+  kAppMemBeg, kAppMemEnd,
+};
 
-void InitializePlatform();
-void FinalizePlatform();
+#else
+# error "Unknown platform"
+#endif
 
 // The additional page is to catch shadow stack overflow as paging fault.
 // Windows wants 64K alignment for mmaps.
@@ -137,18 +236,23 @@ const uptr kTotalTraceSize = (kTraceSize * sizeof(Event) + sizeof(Trace)
     + (64 << 10) + (64 << 10) - 1) & ~((64 << 10) - 1);
 
 uptr ALWAYS_INLINE GetThreadTrace(int tid) {
-  uptr p = kTraceMemBegin + (uptr)tid * kTotalTraceSize;
-  DCHECK_LT(p, kTraceMemBegin + kTraceMemSize);
+  uptr p = kTraceMemBeg + (uptr)tid * kTotalTraceSize;
+  DCHECK_LT(p, kTraceMemEnd);
   return p;
 }
 
 uptr ALWAYS_INLINE GetThreadTraceHeader(int tid) {
-  uptr p = kTraceMemBegin + (uptr)tid * kTotalTraceSize
+  uptr p = kTraceMemBeg + (uptr)tid * kTotalTraceSize
       + kTraceSize * sizeof(Event);
-  DCHECK_LT(p, kTraceMemBegin + kTraceMemSize);
+  DCHECK_LT(p, kTraceMemEnd);
   return p;
 }
 
+void InitializePlatform();
+void FlushShadowMemory();
+void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive);
+uptr GetRSS();
+
 void *internal_start_thread(void(*func)(void*), void *arg);
 void internal_join_thread(void *th);
 
@@ -164,8 +268,4 @@ int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m,
 
 }  // namespace __tsan
 
-#else  // defined(__LP64__) || defined(_WIN64)
-# error "Only 64-bit is supported"
-#endif
-
 #endif  // TSAN_PLATFORM_H