summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/hwasan/.clang-format1
-rw-r--r--lib/hwasan/CMakeLists.txt145
-rw-r--r--lib/hwasan/hwasan.cc301
-rw-r--r--lib/hwasan/hwasan.h176
-rw-r--r--lib/hwasan/hwasan.syms.extra2
-rw-r--r--lib/hwasan/hwasan_allocator.cc330
-rw-r--r--lib/hwasan/hwasan_allocator.h55
-rw-r--r--lib/hwasan/hwasan_blacklist.txt7
-rw-r--r--lib/hwasan/hwasan_flags.h30
-rw-r--r--lib/hwasan/hwasan_flags.inc29
-rw-r--r--lib/hwasan/hwasan_interceptors.cc483
-rw-r--r--lib/hwasan/hwasan_interface_internal.h105
-rw-r--r--lib/hwasan/hwasan_linux.cc194
-rw-r--r--lib/hwasan/hwasan_new_delete.cc66
-rw-r--r--lib/hwasan/hwasan_poisoning.cc36
-rw-r--r--lib/hwasan/hwasan_poisoning.h25
-rw-r--r--lib/hwasan/hwasan_report.cc133
-rw-r--r--lib/hwasan/hwasan_thread.cc75
-rw-r--r--lib/hwasan/hwasan_thread.h81
-rw-r--r--lib/sanitizer_common/sanitizer_internal_defs.h1
20 files changed, 2275 insertions, 0 deletions
diff --git a/lib/hwasan/.clang-format b/lib/hwasan/.clang-format
new file mode 100644
index 000000000..f6cb8ad93
--- /dev/null
+++ b/lib/hwasan/.clang-format
@@ -0,0 +1 @@
+BasedOnStyle: Google
diff --git a/lib/hwasan/CMakeLists.txt b/lib/hwasan/CMakeLists.txt
new file mode 100644
index 000000000..3f3a61555
--- /dev/null
+++ b/lib/hwasan/CMakeLists.txt
@@ -0,0 +1,145 @@
+include_directories(..)
+
+# Runtime library sources and build flags.
+set(HWASAN_RTL_SOURCES
+ hwasan.cc
+ hwasan_allocator.cc
+ hwasan_interceptors.cc
+ hwasan_linux.cc
+ hwasan_report.cc
+ hwasan_thread.cc
+ hwasan_poisoning.cc
+ )
+
+set(HWASAN_RTL_CXX_SOURCES
+ hwasan_new_delete.cc)
+
+
+set(HWASAN_RTL_CFLAGS ${SANITIZER_COMMON_CFLAGS})
+append_rtti_flag(OFF HWASAN_RTL_CFLAGS)
+append_list_if(COMPILER_RT_HAS_FPIE_FLAG -fPIE HWASAN_RTL_CFLAGS)
+# Prevent clang from generating libc calls.
+append_list_if(COMPILER_RT_HAS_FFREESTANDING_FLAG -ffreestanding HWASAN_RTL_CFLAGS)
+
+set(HWASAN_DYNAMIC_LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS})
+
+if(ANDROID)
+# On Android, -z global does not do what it is documented to do.
+# On Android, -z global moves the library ahead in the lookup order,
+# placing it right after the LD_PRELOADs. This is used to compensate for the fact
+# that Android linker does not look at the dependencies of the main executable
+# that aren't dependencies of the current DSO when resolving symbols from said DSO.
+# As a net result, this allows running ASan executables without LD_PRELOAD-ing the
+# ASan runtime library.
+# The above is applicable to L MR1 or newer.
+ if (COMPILER_RT_HAS_Z_GLOBAL)
+ list(APPEND HWASAN_DYNAMIC_LINK_FLAGS -Wl,-z,global)
+ endif()
+endif()
+
+set(HWASAN_DYNAMIC_CFLAGS ${HWASAN_RTL_CFLAGS})
+append_list_if(COMPILER_RT_HAS_FTLS_MODEL_INITIAL_EXEC
+ -ftls-model=initial-exec HWASAN_DYNAMIC_CFLAGS)
+append_list_if(MSVC /DEBUG HWASAN_DYNAMIC_LINK_FLAGS)
+
+set(HWASAN_DYNAMIC_LIBS ${SANITIZER_CXX_ABI_LIBRARY} ${SANITIZER_COMMON_LINK_LIBS})
+
+append_list_if(COMPILER_RT_HAS_LIBDL dl HWASAN_DYNAMIC_LIBS)
+append_list_if(COMPILER_RT_HAS_LIBRT rt HWASAN_DYNAMIC_LIBS)
+append_list_if(COMPILER_RT_HAS_LIBM m HWASAN_DYNAMIC_LIBS)
+append_list_if(COMPILER_RT_HAS_LIBPTHREAD pthread HWASAN_DYNAMIC_LIBS)
+append_list_if(COMPILER_RT_HAS_LIBLOG log HWASAN_DYNAMIC_LIBS)
+
+# Static runtime library.
+add_compiler_rt_component(hwasan)
+
+add_compiler_rt_object_libraries(RTHwasan
+ ARCHS ${HWASAN_SUPPORTED_ARCH}
+ SOURCES ${HWASAN_RTL_SOURCES} CFLAGS ${HWASAN_RTL_CFLAGS})
+add_compiler_rt_object_libraries(RTHwasan_cxx
+ ARCHS ${HWASAN_SUPPORTED_ARCH}
+ SOURCES ${HWASAN_RTL_CXX_SOURCES} CFLAGS ${HWASAN_RTL_CFLAGS})
+add_compiler_rt_object_libraries(RTHwasan_dynamic
+ ARCHS ${HWASAN_SUPPORTED_ARCH}
+ SOURCES ${HWASAN_RTL_SOURCES} ${TSAN_RTL_CXX_SOURCES}
+ CFLAGS ${HWASAN_DYNAMIC_CFLAGS})
+
+file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cc "")
+add_compiler_rt_object_libraries(RTHwasan_dynamic_version_script_dummy
+ ARCHS ${HWASAN_SUPPORTED_ARCH}
+ SOURCES ${CMAKE_CURRENT_BINARY_DIR}/dummy.cc
+ CFLAGS ${HWASAN_DYNAMIC_CFLAGS})
+
+foreach(arch ${HWASAN_SUPPORTED_ARCH})
+ add_compiler_rt_runtime(clang_rt.hwasan
+ STATIC
+ ARCHS ${arch}
+ OBJECT_LIBS RTHwasan
+ RTInterception
+ RTSanitizerCommon
+ RTSanitizerCommonLibc
+ RTUbsan
+ CFLAGS ${HWASAN_RTL_CFLAGS}
+ PARENT_TARGET hwasan)
+ add_compiler_rt_runtime(clang_rt.hwasan_cxx
+ STATIC
+ ARCHS ${arch}
+ OBJECT_LIBS RTHwasan_cxx
+ RTUbsan_cxx
+ CFLAGS ${HWASAN_RTL_CFLAGS}
+ PARENT_TARGET hwasan)
+
+ if (UNIX)
+ add_sanitizer_rt_version_list(clang_rt.hwasan-dynamic-${arch}
+ LIBS clang_rt.hwasan-${arch} clang_rt.hwasan_cxx-${arch}
+ EXTRA hwasan.syms.extra)
+ set(VERSION_SCRIPT_FLAG
+ -Wl,--version-script,${CMAKE_CURRENT_BINARY_DIR}/clang_rt.hwasan-dynamic-${arch}.vers)
+ set_property(SOURCE
+ ${CMAKE_CURRENT_BINARY_DIR}/dummy.cc
+ APPEND PROPERTY
+ OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/clang_rt.hwasan-dynamic-${arch}.vers)
+ else()
+ set(VERSION_SCRIPT_FLAG)
+ endif()
+
+
+ add_compiler_rt_runtime(clang_rt.hwasan
+ SHARED
+ ARCHS ${arch}
+ OBJECT_LIBS
+ RTHwasan_dynamic
+ RTInterception
+ RTSanitizerCommon
+ RTSanitizerCommonLibc
+ RTUbsan
+ # The only purpose of RTHWAsan_dynamic_version_script_dummy is to
+ # carry a dependency of the shared runtime on the version script.
+ # Replacing it with a straightforward
+ # add_dependencies(clang_rt.asan-dynamic-${arch} clang_rt.asan-dynamic-${arch}-version-list)
+ # generates an order-only dependency in ninja.
+ RTHwasan_dynamic_version_script_dummy
+ CFLAGS ${HWASAN_DYNAMIC_CFLAGS}
+ LINK_FLAGS ${HWASAN_DYNAMIC_LINK_FLAGS}
+ ${VERSION_SCRIPT_FLAG}
+ LINK_LIBS ${HWASAN_DYNAMIC_LIBS}
+ DEFS ${ASAN_DYNAMIC_DEFINITIONS}
+ PARENT_TARGET hwasan)
+
+ if(UNIX)
+ add_sanitizer_rt_symbols(clang_rt.hwasan
+ ARCHS ${arch}
+ EXTRA hwasan.syms.extra)
+ add_sanitizer_rt_symbols(clang_rt.hwasan_cxx
+ ARCHS ${arch}
+ EXTRA hwasan.syms.extra)
+ add_dependencies(hwasan clang_rt.hwasan-${arch}-symbols
+ clang_rt.hwasan_cxx-${arch}-symbols)
+ endif()
+endforeach()
+
+add_compiler_rt_resource_file(hwasan_blacklist hwasan_blacklist.txt hwasan)
+
+# if(COMPILER_RT_INCLUDE_TESTS)
+# add_subdirectory(tests)
+# endif()
diff --git a/lib/hwasan/hwasan.cc b/lib/hwasan/hwasan.cc
new file mode 100644
index 000000000..40012c130
--- /dev/null
+++ b/lib/hwasan/hwasan.cc
@@ -0,0 +1,301 @@
+//===-- hwasan.cc -----------------------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// HWAddressSanitizer runtime.
+//===----------------------------------------------------------------------===//
+
+#include "hwasan.h"
+#include "hwasan_thread.h"
+#include "hwasan_poisoning.h"
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_flag_parser.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_procmaps.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "sanitizer_common/sanitizer_symbolizer.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "ubsan/ubsan_flags.h"
+#include "ubsan/ubsan_init.h"
+
+// ACHTUNG! No system header includes in this file.
+
+using namespace __sanitizer;
+
+namespace __hwasan {
+
+void EnterSymbolizer() {
+ HwasanThread *t = GetCurrentThread();
+ CHECK(t);
+ t->EnterSymbolizer();
+}
+void ExitSymbolizer() {
+ HwasanThread *t = GetCurrentThread();
+ CHECK(t);
+ t->LeaveSymbolizer();
+}
+bool IsInSymbolizer() {
+ HwasanThread *t = GetCurrentThread();
+ return t && t->InSymbolizer();
+}
+
+static Flags hwasan_flags;
+
+Flags *flags() {
+ return &hwasan_flags;
+}
+
+int hwasan_inited = 0;
+bool hwasan_init_is_running;
+
+int hwasan_report_count = 0;
+
+void Flags::SetDefaults() {
+#define HWASAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
+#include "hwasan_flags.inc"
+#undef HWASAN_FLAG
+}
+
+static void RegisterHwasanFlags(FlagParser *parser, Flags *f) {
+#define HWASAN_FLAG(Type, Name, DefaultValue, Description) \
+ RegisterFlag(parser, #Name, Description, &f->Name);
+#include "hwasan_flags.inc"
+#undef HWASAN_FLAG
+}
+
+static void InitializeFlags() {
+ SetCommonFlagsDefaults();
+ {
+ CommonFlags cf;
+ cf.CopyFrom(*common_flags());
+ cf.external_symbolizer_path = GetEnv("HWASAN_SYMBOLIZER_PATH");
+ cf.malloc_context_size = 20;
+ cf.handle_ioctl = true;
+ // FIXME: test and enable.
+ cf.check_printf = false;
+ cf.intercept_tls_get_addr = true;
+ cf.exitcode = 99;
+ cf.handle_sigill = kHandleSignalExclusive;
+ OverrideCommonFlags(cf);
+ }
+
+ Flags *f = flags();
+ f->SetDefaults();
+
+ FlagParser parser;
+ RegisterHwasanFlags(&parser, f);
+ RegisterCommonFlags(&parser);
+
+#if HWASAN_CONTAINS_UBSAN
+ __ubsan::Flags *uf = __ubsan::flags();
+ uf->SetDefaults();
+
+ FlagParser ubsan_parser;
+ __ubsan::RegisterUbsanFlags(&ubsan_parser, uf);
+ RegisterCommonFlags(&ubsan_parser);
+#endif
+
+ // Override from user-specified string.
+ if (__hwasan_default_options)
+ parser.ParseString(__hwasan_default_options());
+#if HWASAN_CONTAINS_UBSAN
+ const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions();
+ ubsan_parser.ParseString(ubsan_default_options);
+#endif
+
+ const char *hwasan_options = GetEnv("HWASAN_OPTIONS");
+ parser.ParseString(hwasan_options);
+#if HWASAN_CONTAINS_UBSAN
+ ubsan_parser.ParseString(GetEnv("UBSAN_OPTIONS"));
+#endif
+ VPrintf(1, "HWASAN_OPTIONS: %s\n", hwasan_options ? hwasan_options : "<empty>");
+
+ InitializeCommonFlags();
+
+ if (Verbosity()) ReportUnrecognizedFlags();
+
+ if (common_flags()->help) parser.PrintFlagDescriptions();
+}
+
+void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp,
+ void *context, bool request_fast_unwind) {
+ HwasanThread *t = GetCurrentThread();
+ if (!t || !StackTrace::WillUseFastUnwind(request_fast_unwind)) {
+ // Block reports from our interceptors during _Unwind_Backtrace.
+ SymbolizerScope sym_scope;
+ return stack->Unwind(max_s, pc, bp, context, 0, 0, request_fast_unwind);
+ }
+ stack->Unwind(max_s, pc, bp, context, t->stack_top(), t->stack_bottom(),
+ request_fast_unwind);
+}
+
+void PrintWarning(uptr pc, uptr bp) {
+ GET_FATAL_STACK_TRACE_PC_BP(pc, bp);
+ ReportInvalidAccess(&stack, 0);
+}
+
+} // namespace __hwasan
+
+// Interface.
+
+using namespace __hwasan;
+
+void __hwasan_init() {
+ CHECK(!hwasan_init_is_running);
+ if (hwasan_inited) return;
+ hwasan_init_is_running = 1;
+ SanitizerToolName = "HWAddressSanitizer";
+
+ InitTlsSize();
+
+ CacheBinaryName();
+ InitializeFlags();
+
+ __sanitizer_set_report_path(common_flags()->log_path);
+
+ InitializeInterceptors();
+ InstallDeadlySignalHandlers(HwasanOnDeadlySignal);
+ InstallAtExitHandler(); // Needs __cxa_atexit interceptor.
+
+ DisableCoreDumperIfNecessary();
+ if (!InitShadow()) {
+ Printf("FATAL: HWAddressSanitizer can not mmap the shadow memory.\n");
+ Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n");
+ Printf("FATAL: Disabling ASLR is known to cause this error.\n");
+ Printf("FATAL: If running under GDB, try "
+ "'set disable-randomization off'.\n");
+ DumpProcessMap();
+ Die();
+ }
+
+ Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer);
+
+ InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
+
+ HwasanTSDInit(HwasanTSDDtor);
+
+ HwasanAllocatorInit();
+
+ HwasanThread *main_thread = HwasanThread::Create(nullptr, nullptr);
+ SetCurrentThread(main_thread);
+ main_thread->ThreadStart();
+
+#if HWASAN_CONTAINS_UBSAN
+ __ubsan::InitAsPlugin();
+#endif
+
+ VPrintf(1, "HWAddressSanitizer init done\n");
+
+ hwasan_init_is_running = 0;
+ hwasan_inited = 1;
+}
+
+void __hwasan_print_shadow(const void *x, uptr size) {
+ // FIXME:
+ Printf("FIXME: __hwasan_print_shadow unimplemented\n");
+}
+
+sptr __hwasan_test_shadow(const void *p, uptr sz) {
+ if (sz == 0)
+ return -1;
+ tag_t ptr_tag = GetTagFromPointer((uptr)p);
+ if (ptr_tag == 0)
+ return -1;
+ uptr ptr_raw = GetAddressFromPointer((uptr)p);
+ uptr shadow_first = MEM_TO_SHADOW(ptr_raw);
+ uptr shadow_last = MEM_TO_SHADOW(ptr_raw + sz - 1);
+ for (uptr s = shadow_first; s <= shadow_last; ++s)
+ if (*(tag_t*)s != ptr_tag)
+ return SHADOW_TO_MEM(s) - ptr_raw;
+ return -1;
+}
+
+u16 __sanitizer_unaligned_load16(const uu16 *p) {
+ return *p;
+}
+u32 __sanitizer_unaligned_load32(const uu32 *p) {
+ return *p;
+}
+u64 __sanitizer_unaligned_load64(const uu64 *p) {
+ return *p;
+}
+void __sanitizer_unaligned_store16(uu16 *p, u16 x) {
+ *p = x;
+}
+void __sanitizer_unaligned_store32(uu32 *p, u32 x) {
+ *p = x;
+}
+void __sanitizer_unaligned_store64(uu64 *p, u64 x) {
+ *p = x;
+}
+
+__attribute__((always_inline))
+static void SigIll() {
+#if defined(__aarch64__)
+ asm("hlt #0x1\n\t");
+#elif defined(__x86_64__) || defined(__i386__)
+ asm("ud2\n\t");
+#else
+ // FIXME: not always sigill.
+ __builtin_trap();
+#endif
+ // __builtin_unreachable();
+}
+
+__attribute__((always_inline, nodebug))
+static void CheckAddress(uptr p) {
+ tag_t ptr_tag = GetTagFromPointer(p);
+ uptr ptr_raw = p & ~kAddressTagMask;
+ tag_t mem_tag = *(tag_t *)MEM_TO_SHADOW(ptr_raw);
+ if (ptr_tag != mem_tag)
+ SigIll();
+}
+
+__attribute__((always_inline, nodebug))
+static void CheckAddressSized(uptr p, uptr sz) {
+ CHECK_NE(0, sz);
+ tag_t ptr_tag = GetTagFromPointer(p);
+ uptr ptr_raw = p & ~kAddressTagMask;
+ tag_t *shadow_first = (tag_t *)MEM_TO_SHADOW(ptr_raw);
+ tag_t *shadow_last = (tag_t *)MEM_TO_SHADOW(ptr_raw + sz - 1);
+ for (tag_t *t = shadow_first; t <= shadow_last; ++t)
+ if (ptr_tag != *t) SigIll();
+}
+
+void __hwasan_load(uptr p, uptr sz) { CheckAddressSized(p, sz); }
+void __hwasan_load1(uptr p) { CheckAddress(p); }
+void __hwasan_load2(uptr p) { CheckAddress(p); }
+void __hwasan_load4(uptr p) { CheckAddress(p); }
+void __hwasan_load8(uptr p) { CheckAddress(p); }
+void __hwasan_load16(uptr p) { CheckAddress(p); }
+
+void __hwasan_store(uptr p, uptr sz) { CheckAddressSized(p, sz); }
+void __hwasan_store1(uptr p) { CheckAddress(p); }
+void __hwasan_store2(uptr p) { CheckAddress(p); }
+void __hwasan_store4(uptr p) { CheckAddress(p); }
+void __hwasan_store8(uptr p) { CheckAddress(p); }
+void __hwasan_store16(uptr p) { CheckAddress(p); }
+
+#if !SANITIZER_SUPPORTS_WEAK_HOOKS
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+const char* __hwasan_default_options() { return ""; }
+} // extern "C"
+#endif
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_print_stack_trace() {
+ GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME());
+ stack.Print();
+}
+} // extern "C"
diff --git a/lib/hwasan/hwasan.h b/lib/hwasan/hwasan.h
new file mode 100644
index 000000000..8ced45e32
--- /dev/null
+++ b/lib/hwasan/hwasan.h
@@ -0,0 +1,176 @@
+//===-- hwasan.h --------------------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Private Hwasan header.
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_H
+#define HWASAN_H
+
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "hwasan_interface_internal.h"
+#include "hwasan_flags.h"
+#include "ubsan/ubsan_platform.h"
+
+#ifndef HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE
+# define HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE 1
+#endif
+
+#ifndef HWASAN_CONTAINS_UBSAN
+# define HWASAN_CONTAINS_UBSAN CAN_SANITIZE_UB
+#endif
+
+typedef u8 tag_t;
+
+// Reasonable values are 4 (for 1/16th shadow) and 6 (for 1/64th).
+const uptr kShadowScale = 4;
+const uptr kShadowAlignment = 1UL << kShadowScale;
+
+#define MEM_TO_SHADOW_OFFSET(mem) ((uptr)(mem) >> kShadowScale)
+#define MEM_TO_SHADOW(mem) \
+ (((uptr)(mem) >> kShadowScale) + \
+ __hwasan_shadow_memory_dynamic_address_internal)
+#define SHADOW_TO_MEM(shadow) \
+ (((uptr)(shadow)-__hwasan_shadow_memory_dynamic_address_internal) \
+ << kShadowScale)
+
+#define MEM_IS_APP(mem) true
+
+// TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address
+// translation and can be used to store a tag.
+const unsigned kAddressTagShift = 56;
+const uptr kAddressTagMask = 0xFFUL << kAddressTagShift;
+
+static inline tag_t GetTagFromPointer(uptr p) {
+ return p >> kAddressTagShift;
+}
+
+static inline uptr GetAddressFromPointer(uptr p) {
+ return p & ~kAddressTagMask;
+}
+
+static inline void * GetAddressFromPointer(const void *p) {
+ return (void *)((uptr)p & ~kAddressTagMask);
+}
+
+static inline uptr AddTagToPointer(uptr p, tag_t tag) {
+ return (p & ~kAddressTagMask) | ((uptr)tag << kAddressTagShift);
+}
+
+namespace __hwasan {
+
+extern int hwasan_inited;
+extern bool hwasan_init_is_running;
+extern int hwasan_report_count;
+
+bool ProtectRange(uptr beg, uptr end);
+bool InitShadow();
+char *GetProcSelfMaps();
+void InitializeInterceptors();
+
+void HwasanAllocatorInit();
+void HwasanAllocatorThreadFinish();
+void HwasanDeallocate(StackTrace *stack, void *ptr);
+
+void *hwasan_malloc(uptr size, StackTrace *stack);
+void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack);
+void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack);
+void *hwasan_valloc(uptr size, StackTrace *stack);
+void *hwasan_pvalloc(uptr size, StackTrace *stack);
+void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack);
+void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack);
+int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size,
+ StackTrace *stack);
+
+void InstallTrapHandler();
+void InstallAtExitHandler();
+
+const char *GetStackOriginDescr(u32 id, uptr *pc);
+
+void EnterSymbolizer();
+void ExitSymbolizer();
+bool IsInSymbolizer();
+
+struct SymbolizerScope {
+ SymbolizerScope() { EnterSymbolizer(); }
+ ~SymbolizerScope() { ExitSymbolizer(); }
+};
+
+void PrintWarning(uptr pc, uptr bp);
+
+void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp,
+ void *context, bool request_fast_unwind);
+
+void ReportInvalidAccess(StackTrace *stack, u32 origin);
+void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size,
+ bool is_store);
+void ReportStats();
+void ReportAtExitStatistics();
+void DescribeMemoryRange(const void *x, uptr size);
+void ReportInvalidAccessInsideAddressRange(const char *what, const void *start, uptr size,
+ uptr offset);
+
+// Returns a "chained" origin id, pointing to the given stack trace followed by
+// the previous origin id.
+u32 ChainOrigin(u32 id, StackTrace *stack);
+
+const int STACK_TRACE_TAG_POISON = StackTrace::TAG_CUSTOM + 1;
+
+#define GET_MALLOC_STACK_TRACE \
+ BufferedStackTrace stack; \
+ if (hwasan_inited) \
+ GetStackTrace(&stack, common_flags()->malloc_context_size, \
+ StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, \
+ common_flags()->fast_unwind_on_malloc)
+
+#define GET_FATAL_STACK_TRACE_PC_BP(pc, bp) \
+ BufferedStackTrace stack; \
+ if (hwasan_inited) \
+ GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr, \
+ common_flags()->fast_unwind_on_fatal)
+
+class ScopedThreadLocalStateBackup {
+ public:
+ ScopedThreadLocalStateBackup() { Backup(); }
+ ~ScopedThreadLocalStateBackup() { Restore(); }
+ void Backup();
+ void Restore();
+ private:
+ u64 va_arg_overflow_size_tls;
+};
+
+void HwasanTSDInit(void (*destructor)(void *tsd));
+void *HwasanTSDGet();
+void HwasanTSDSet(void *tsd);
+void HwasanTSDDtor(void *tsd);
+
+void HwasanOnDeadlySignal(int signo, void *info, void *context);
+
+} // namespace __hwasan
+
+#define HWASAN_MALLOC_HOOK(ptr, size) \
+ do { \
+ if (&__sanitizer_malloc_hook) { \
+ __sanitizer_malloc_hook(ptr, size); \
+ } \
+ RunMallocHooks(ptr, size); \
+ } while (false)
+#define HWASAN_FREE_HOOK(ptr) \
+ do { \
+ if (&__sanitizer_free_hook) { \
+ __sanitizer_free_hook(ptr); \
+ } \
+ RunFreeHooks(ptr); \
+ } while (false)
+
+#endif // HWASAN_H
diff --git a/lib/hwasan/hwasan.syms.extra b/lib/hwasan/hwasan.syms.extra
new file mode 100644
index 000000000..8738627e0
--- /dev/null
+++ b/lib/hwasan/hwasan.syms.extra
@@ -0,0 +1,2 @@
+__hwasan_*
+__ubsan_*
diff --git a/lib/hwasan/hwasan_allocator.cc b/lib/hwasan/hwasan_allocator.cc
new file mode 100644
index 000000000..fbcaf78b8
--- /dev/null
+++ b/lib/hwasan/hwasan_allocator.cc
@@ -0,0 +1,330 @@
+//===-- hwasan_allocator.cc --------------------------- ---------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// HWAddressSanitizer allocator.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_checks.h"
+#include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_errno.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "hwasan.h"
+#include "hwasan_allocator.h"
+#include "hwasan_thread.h"
+#include "hwasan_poisoning.h"
+
+namespace __hwasan {
+
+enum {
+ CHUNK_INVALID = 0,
+ CHUNK_FREE = 1,
+ CHUNK_ALLOCATED = 2
+};
+
+struct Metadata {
+ u64 state : 2;
+ u64 requested_size : 62;
+ u32 alloc_context_id;
+ u32 free_context_id;
+};
+
+bool HwasanChunkView::IsValid() const {
+ return metadata_ && metadata_->state != CHUNK_INVALID;
+}
+bool HwasanChunkView::IsAllocated() const {
+ return metadata_ && metadata_->state == CHUNK_ALLOCATED;
+}
+uptr HwasanChunkView::Beg() const {
+ return block_;
+}
+uptr HwasanChunkView::End() const {
+ return Beg() + UsedSize();
+}
+uptr HwasanChunkView::UsedSize() const {
+ return metadata_->requested_size;
+}
+u32 HwasanChunkView::GetAllocStackId() const {
+ return metadata_->alloc_context_id;
+}
+u32 HwasanChunkView::GetFreeStackId() const {
+ return metadata_->free_context_id;
+}
+
+struct HwasanMapUnmapCallback {
+ void OnMap(uptr p, uptr size) const {}
+ void OnUnmap(uptr p, uptr size) const {
+ // We are about to unmap a chunk of user memory.
+ // It can return as user-requested mmap() or another thread stack.
+ // Make it accessible with zero-tagged pointer.
+ TagMemory(p, size, 0);
+ }
+};
+
+#if !defined(__aarch64__)
+#error unsupported platform
+#endif
+
+static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G
+static const uptr kRegionSizeLog = 20;
+static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog;
+typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap;
+
+struct AP32 {
+ static const uptr kSpaceBeg = 0;
+ static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
+ static const uptr kMetadataSize = sizeof(Metadata);
+ typedef __sanitizer::CompactSizeClassMap SizeClassMap;
+ static const uptr kRegionSizeLog = __hwasan::kRegionSizeLog;
+ typedef __hwasan::ByteMap ByteMap;
+ typedef HwasanMapUnmapCallback MapUnmapCallback;
+ static const uptr kFlags = 0;
+};
+typedef SizeClassAllocator32<AP32> PrimaryAllocator;
+typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache;
+typedef LargeMmapAllocator<HwasanMapUnmapCallback> SecondaryAllocator;
+typedef CombinedAllocator<PrimaryAllocator, AllocatorCache,
+ SecondaryAllocator> Allocator;
+
+static Allocator allocator;
+static AllocatorCache fallback_allocator_cache;
+static SpinMutex fallback_mutex;
+static atomic_uint8_t hwasan_allocator_tagging_enabled;
+
+void HwasanAllocatorInit() {
+ atomic_store_relaxed(&hwasan_allocator_tagging_enabled,
+ !flags()->disable_allocator_tagging);
+ SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
+ allocator.Init(common_flags()->allocator_release_to_os_interval_ms);
+}
+
+AllocatorCache *GetAllocatorCache(HwasanThreadLocalMallocStorage *ms) {
+ CHECK(ms);
+ CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache));
+ return reinterpret_cast<AllocatorCache *>(ms->allocator_cache);
+}
+
+void HwasanThreadLocalMallocStorage::CommitBack() {
+ allocator.SwallowCache(GetAllocatorCache(this));
+}
+
+static void *HwasanAllocate(StackTrace *stack, uptr size, uptr alignment,
+ bool zeroise) {
+ alignment = Max(alignment, kShadowAlignment);
+ size = RoundUpTo(size, kShadowAlignment);
+
+ if (size > kMaxAllowedMallocSize) {
+ Report("WARNING: HWAddressSanitizer failed to allocate %p bytes\n",
+ (void *)size);
+ return Allocator::FailureHandler::OnBadRequest();
+ }
+ HwasanThread *t = GetCurrentThread();
+ void *allocated;
+ if (t) {
+ AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
+ allocated = allocator.Allocate(cache, size, alignment);
+ } else {
+ SpinMutexLock l(&fallback_mutex);
+ AllocatorCache *cache = &fallback_allocator_cache;
+ allocated = allocator.Allocate(cache, size, alignment);
+ }
+ Metadata *meta =
+ reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated));
+ meta->state = CHUNK_ALLOCATED;
+ meta->requested_size = size;
+ meta->alloc_context_id = StackDepotPut(*stack);
+ if (zeroise)
+ internal_memset(allocated, 0, size);
+
+ void *user_ptr = (flags()->tag_in_malloc &&
+ atomic_load_relaxed(&hwasan_allocator_tagging_enabled))
+ ? (void *)TagMemoryAligned((uptr)allocated, size, 0xBB)
+ : allocated;
+
+ HWASAN_MALLOC_HOOK(user_ptr, size);
+ return user_ptr;
+}
+
+void HwasanDeallocate(StackTrace *stack, void *user_ptr) {
+ CHECK(user_ptr);
+ HWASAN_FREE_HOOK(user_ptr);
+
+ void *p = GetAddressFromPointer(user_ptr);
+ Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p));
+ uptr size = meta->requested_size;
+ meta->state = CHUNK_FREE;
+ meta->requested_size = 0;
+ meta->free_context_id = StackDepotPut(*stack);
+ // This memory will not be reused by anyone else, so we are free to keep it
+ // poisoned.
+ if (flags()->tag_in_free &&
+ atomic_load_relaxed(&hwasan_allocator_tagging_enabled))
+ TagMemoryAligned((uptr)p, size, 0xBC);
+ HwasanThread *t = GetCurrentThread();
+ if (t) {
+ AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
+ allocator.Deallocate(cache, p);
+ } else {
+ SpinMutexLock l(&fallback_mutex);
+ AllocatorCache *cache = &fallback_allocator_cache;
+ allocator.Deallocate(cache, p);
+ }
+}
+
+void *HwasanReallocate(StackTrace *stack, void *user_old_p, uptr new_size,
+ uptr alignment) {
+ alignment = Max(alignment, kShadowAlignment);
+ new_size = RoundUpTo(new_size, kShadowAlignment);
+
+ void *old_p = GetAddressFromPointer(user_old_p);
+ Metadata *meta = reinterpret_cast<Metadata*>(allocator.GetMetaData(old_p));
+ uptr old_size = meta->requested_size;
+ uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(old_p);
+ if (new_size <= actually_allocated_size) {
+ // We are not reallocating here.
+ // FIXME: update stack trace for the allocation?
+ meta->requested_size = new_size;
+ if (!atomic_load_relaxed(&hwasan_allocator_tagging_enabled))
+ return user_old_p;
+ if (flags()->retag_in_realloc)
+ return (void *)TagMemoryAligned((uptr)old_p, new_size, 0xCC);
+ if (new_size > old_size) {
+ tag_t tag = GetTagFromPointer((uptr)user_old_p);
+ TagMemoryAligned((uptr)old_p + old_size, new_size - old_size, tag);
+ }
+ return user_old_p;
+ }
+ uptr memcpy_size = Min(new_size, old_size);
+ void *new_p = HwasanAllocate(stack, new_size, alignment, false /*zeroise*/);
+ if (new_p) {
+ internal_memcpy(new_p, old_p, memcpy_size);
+ HwasanDeallocate(stack, old_p);
+ }
+ return new_p;
+}
+
+HwasanChunkView FindHeapChunkByAddress(uptr address) {
+ void *block = allocator.GetBlockBegin(reinterpret_cast<void*>(address));
+ if (!block)
+ return HwasanChunkView();
+ Metadata *metadata =
+ reinterpret_cast<Metadata*>(allocator.GetMetaData(block));
+ return HwasanChunkView(reinterpret_cast<uptr>(block), metadata);
+}
+
+static uptr AllocationSize(const void *user_ptr) {
+ const void *p = GetAddressFromPointer(user_ptr);
+ if (!p) return 0;
+ const void *beg = allocator.GetBlockBegin(p);
+ if (beg != p) return 0;
+ Metadata *b = (Metadata *)allocator.GetMetaData(p);
+ return b->requested_size;
+}
+
+void *hwasan_malloc(uptr size, StackTrace *stack) {
+ return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false));
+}
+
+void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack) {
+ if (UNLIKELY(CheckForCallocOverflow(size, nmemb)))
+ return SetErrnoOnNull(Allocator::FailureHandler::OnBadRequest());
+ return SetErrnoOnNull(HwasanAllocate(stack, nmemb * size, sizeof(u64), true));
+}
+
+void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack) {
+ if (!ptr)
+ return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false));
+ if (size == 0) {
+ HwasanDeallocate(stack, ptr);
+ return nullptr;
+ }
+ return SetErrnoOnNull(HwasanReallocate(stack, ptr, size, sizeof(u64)));
+}
+
+void *hwasan_valloc(uptr size, StackTrace *stack) {
+ return SetErrnoOnNull(HwasanAllocate(stack, size, GetPageSizeCached(), false));
+}
+
+void *hwasan_pvalloc(uptr size, StackTrace *stack) {
+ uptr PageSize = GetPageSizeCached();
+ if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
+ errno = errno_ENOMEM;
+ return Allocator::FailureHandler::OnBadRequest();
+ }
+ // pvalloc(0) should allocate one page.
+ size = size ? RoundUpTo(size, PageSize) : PageSize;
+ return SetErrnoOnNull(HwasanAllocate(stack, size, PageSize, false));
+}
+
+void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack) {
+ if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
+ errno = errno_EINVAL;
+ return Allocator::FailureHandler::OnBadRequest();
+ }
+ return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, false));
+}
+
+void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack) {
+ if (UNLIKELY(!IsPowerOfTwo(alignment))) {
+ errno = errno_EINVAL;
+ return Allocator::FailureHandler::OnBadRequest();
+ }
+ return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, false));
+}
+
+int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size,
+ StackTrace *stack) {
+ if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
+ Allocator::FailureHandler::OnBadRequest();
+ return errno_EINVAL;
+ }
+ void *ptr = HwasanAllocate(stack, size, alignment, false);
+ if (UNLIKELY(!ptr))
+ return errno_ENOMEM;
+ CHECK(IsAligned((uptr)ptr, alignment));
+ *memptr = ptr;
+ return 0;
+}
+
+} // namespace __hwasan
+
+using namespace __hwasan;
+
+void __hwasan_enable_allocator_tagging() {
+ atomic_store_relaxed(&hwasan_allocator_tagging_enabled, 1);
+}
+
+void __hwasan_disable_allocator_tagging() {
+ atomic_store_relaxed(&hwasan_allocator_tagging_enabled, 0);
+}
+
+uptr __sanitizer_get_current_allocated_bytes() {
+ uptr stats[AllocatorStatCount];
+ allocator.GetStats(stats);
+ return stats[AllocatorStatAllocated];
+}
+
+uptr __sanitizer_get_heap_size() {
+ uptr stats[AllocatorStatCount];
+ allocator.GetStats(stats);
+ return stats[AllocatorStatMapped];
+}
+
+uptr __sanitizer_get_free_bytes() { return 1; }
+
+uptr __sanitizer_get_unmapped_bytes() { return 1; }
+
+uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
+
+int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }
+
+uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }
diff --git a/lib/hwasan/hwasan_allocator.h b/lib/hwasan/hwasan_allocator.h
new file mode 100644
index 000000000..d025112e9
--- /dev/null
+++ b/lib/hwasan/hwasan_allocator.h
@@ -0,0 +1,55 @@
+//===-- hwasan_allocator.h ----------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_ALLOCATOR_H
+#define HWASAN_ALLOCATOR_H
+
+#include "sanitizer_common/sanitizer_common.h"
+
+namespace __hwasan {
+
+struct HwasanThreadLocalMallocStorage {
+ uptr quarantine_cache[16];
+ // Allocator cache contains atomic_uint64_t which must be 8-byte aligned.
+ ALIGNED(8) uptr allocator_cache[96 * (512 * 8 + 16)]; // Opaque.
+ void CommitBack();
+
+ private:
+ // These objects are allocated via mmap() and are zero-initialized.
+ HwasanThreadLocalMallocStorage() {}
+};
+
+struct Metadata;
+
+class HwasanChunkView {
+ public:
+ HwasanChunkView() : block_(0), metadata_(nullptr) {}
+ HwasanChunkView(uptr block, Metadata *metadata)
+ : block_(block), metadata_(metadata) {}
+ bool IsValid() const; // Checks if it points to a valid allocated chunk
+ bool IsAllocated() const; // Checks if the memory is currently allocated
+ uptr Beg() const; // First byte of user memory
+ uptr End() const; // Last byte of user memory
+ uptr UsedSize() const; // Size requested by the user
+ u32 GetAllocStackId() const;
+ u32 GetFreeStackId() const;
+ private:
+ uptr block_;
+ Metadata *const metadata_;
+};
+
+HwasanChunkView FindHeapChunkByAddress(uptr address);
+
+} // namespace __hwasan
+
+#endif // HWASAN_ALLOCATOR_H
diff --git a/lib/hwasan/hwasan_blacklist.txt b/lib/hwasan/hwasan_blacklist.txt
new file mode 100644
index 000000000..395ba28f0
--- /dev/null
+++ b/lib/hwasan/hwasan_blacklist.txt
@@ -0,0 +1,7 @@
+# Blacklist for HWAddressSanitizer. Turns off instrumentation of particular
+# functions or sources. Use with care. You may set location of blacklist
+# at compile-time using -fsanitize-blacklist=<path> flag.
+
+# Example usage:
+# fun:*bad_function_name*
+# src:file_with_tricky_code.cc
diff --git a/lib/hwasan/hwasan_flags.h b/lib/hwasan/hwasan_flags.h
new file mode 100644
index 000000000..16d60c4d8
--- /dev/null
+++ b/lib/hwasan/hwasan_flags.h
@@ -0,0 +1,30 @@
+//===-- hwasan_flags.h --------------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+#ifndef HWASAN_FLAGS_H
+#define HWASAN_FLAGS_H
+
+namespace __hwasan {
+
+struct Flags {
+#define HWASAN_FLAG(Type, Name, DefaultValue, Description) Type Name;
+#include "hwasan_flags.inc"
+#undef HWASAN_FLAG
+
+ void SetDefaults();
+};
+
+Flags *flags();
+
+} // namespace __hwasan
+
+#endif // HWASAN_FLAGS_H
diff --git a/lib/hwasan/hwasan_flags.inc b/lib/hwasan/hwasan_flags.inc
new file mode 100644
index 000000000..a2cb701ae
--- /dev/null
+++ b/lib/hwasan/hwasan_flags.inc
@@ -0,0 +1,29 @@
+//===-- hwasan_flags.inc ------------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Hwasan runtime flags.
+//
+//===----------------------------------------------------------------------===//
+#ifndef HWASAN_FLAG
+# error "Define HWASAN_FLAG prior to including this file!"
+#endif
+
+// HWASAN_FLAG(Type, Name, DefaultValue, Description)
+// See COMMON_FLAG in sanitizer_flags.inc for more details.
+
+HWASAN_FLAG(bool, tag_in_malloc, true, "")
+HWASAN_FLAG(bool, tag_in_free, true, "")
+HWASAN_FLAG(bool, retag_in_realloc, true, "")
+HWASAN_FLAG(bool, print_stats, false, "")
+HWASAN_FLAG(bool, halt_on_error, true, "")
+HWASAN_FLAG(bool, atexit, false, "")
+
+// Test only flag to disable malloc/realloc/free memory tagging on startup.
+// Tagging can be reenabled with __hwasan_enable_allocator_tagging().
+HWASAN_FLAG(bool, disable_allocator_tagging, false, "")
diff --git a/lib/hwasan/hwasan_interceptors.cc b/lib/hwasan/hwasan_interceptors.cc
new file mode 100644
index 000000000..fb39e9fda
--- /dev/null
+++ b/lib/hwasan/hwasan_interceptors.cc
@@ -0,0 +1,483 @@
+//===-- hwasan_interceptors.cc ----------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Interceptors for standard library functions.
+//
+// FIXME: move as many interceptors as possible into
+// sanitizer_common/sanitizer_common_interceptors.h
+//===----------------------------------------------------------------------===//
+
+#include "interception/interception.h"
+#include "hwasan.h"
+#include "hwasan_thread.h"
+#include "hwasan_poisoning.h"
+#include "sanitizer_common/sanitizer_platform_limits_posix.h"
+#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_allocator_internal.h"
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_errno.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_linux.h"
+#include "sanitizer_common/sanitizer_tls_get_addr.h"
+
+#include <stdarg.h>
+// ACHTUNG! No other system header includes in this file.
+// Ideally, we should get rid of stdarg.h as well.
+
+using namespace __hwasan;
+
+using __sanitizer::memory_order;
+using __sanitizer::atomic_load;
+using __sanitizer::atomic_store;
+using __sanitizer::atomic_uintptr_t;
+
+DECLARE_REAL(SIZE_T, strlen, const char *s)
+DECLARE_REAL(SIZE_T, strnlen, const char *s, SIZE_T maxlen)
+DECLARE_REAL(void *, memcpy, void *dest, const void *src, uptr n)
+DECLARE_REAL(void *, memset, void *dest, int c, uptr n)
+
+bool IsInInterceptorScope() {
+ HwasanThread *t = GetCurrentThread();
+ return t && t->InInterceptorScope();
+}
+
+struct InterceptorScope {
+ InterceptorScope() {
+ HwasanThread *t = GetCurrentThread();
+ if (t)
+ t->EnterInterceptorScope();
+ }
+ ~InterceptorScope() {
+ HwasanThread *t = GetCurrentThread();
+ if (t)
+ t->LeaveInterceptorScope();
+ }
+};
+
+static uptr allocated_for_dlsym;
+static const uptr kDlsymAllocPoolSize = 1024;
+static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
+
+static bool IsInDlsymAllocPool(const void *ptr) {
+ uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
+ return off < sizeof(alloc_memory_for_dlsym);
+}
+
+static void *AllocateFromLocalPool(uptr size_in_bytes) {
+ uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
+ void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
+ allocated_for_dlsym += size_in_words;
+ CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
+ return mem;
+}
+
+#define ENSURE_HWASAN_INITED() do { \
+ CHECK(!hwasan_init_is_running); \
+ if (!hwasan_inited) { \
+ __hwasan_init(); \
+ } \
+} while (0)
+
+
+
+#define HWASAN_READ_RANGE(ctx, offset, size) \
+ CHECK_UNPOISONED(offset, size)
+#define HWASAN_WRITE_RANGE(ctx, offset, size) \
+ CHECK_UNPOISONED(offset, size)
+
+
+
+// Check that [x, x+n) range is unpoisoned.
+#define CHECK_UNPOISONED_0(x, n) \
+ do { \
+ sptr __offset = __hwasan_test_shadow(x, n); \
+ if (__hwasan::IsInSymbolizer()) break; \
+ if (__offset >= 0) { \
+ GET_CALLER_PC_BP_SP; \
+ (void)sp; \
+ ReportInvalidAccessInsideAddressRange(__func__, x, n, __offset); \
+ __hwasan::PrintWarning(pc, bp); \
+ if (__hwasan::flags()->halt_on_error) { \
+ Printf("Exiting\n"); \
+ Die(); \
+ } \
+ } \
+ } while (0)
+
+// Check that [x, x+n) range is unpoisoned unless we are in a nested
+// interceptor.
+#define CHECK_UNPOISONED(x, n) \
+ do { \
+ if (!IsInInterceptorScope()) CHECK_UNPOISONED_0(x, n); \
+ } while (0)
+
+#define CHECK_UNPOISONED_STRING_OF_LEN(x, len, n) \
+ CHECK_UNPOISONED((x), \
+ common_flags()->strict_string_checks ? (len) + 1 : (n) )
+
+
+INTERCEPTOR(int, posix_memalign, void **memptr, SIZE_T alignment, SIZE_T size) {
+ GET_MALLOC_STACK_TRACE;
+ CHECK_NE(memptr, 0);
+ int res = hwasan_posix_memalign(memptr, alignment, size, &stack);
+ return res;
+}
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR(void *, memalign, SIZE_T alignment, SIZE_T size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_memalign(alignment, size, &stack);
+}
+#define HWASAN_MAYBE_INTERCEPT_MEMALIGN INTERCEPT_FUNCTION(memalign)
+#else
+#define HWASAN_MAYBE_INTERCEPT_MEMALIGN
+#endif
+
+INTERCEPTOR(void *, aligned_alloc, SIZE_T alignment, SIZE_T size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_aligned_alloc(alignment, size, &stack);
+}
+
+INTERCEPTOR(void *, __libc_memalign, SIZE_T alignment, SIZE_T size) {
+ GET_MALLOC_STACK_TRACE;
+ void *ptr = hwasan_memalign(alignment, size, &stack);
+ if (ptr)
+ DTLS_on_libc_memalign(ptr, size);
+ return ptr;
+}
+
+INTERCEPTOR(void *, valloc, SIZE_T size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_valloc(size, &stack);
+}
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR(void *, pvalloc, SIZE_T size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_pvalloc(size, &stack);
+}
+#define HWASAN_MAYBE_INTERCEPT_PVALLOC INTERCEPT_FUNCTION(pvalloc)
+#else
+#define HWASAN_MAYBE_INTERCEPT_PVALLOC
+#endif
+
+INTERCEPTOR(void, free, void *ptr) {
+ GET_MALLOC_STACK_TRACE;
+ if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
+ HwasanDeallocate(&stack, ptr);
+}
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR(void, cfree, void *ptr) {
+ GET_MALLOC_STACK_TRACE;
+ if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
+ HwasanDeallocate(&stack, ptr);
+}
+#define HWASAN_MAYBE_INTERCEPT_CFREE INTERCEPT_FUNCTION(cfree)
+#else
+#define HWASAN_MAYBE_INTERCEPT_CFREE
+#endif
+
+INTERCEPTOR(uptr, malloc_usable_size, void *ptr) {
+ return __sanitizer_get_allocated_size(ptr);
+}
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+// This function actually returns a struct by value, but we can't unpoison a
+// temporary! The following is equivalent on all supported platforms but
+// aarch64 (which uses a different register for sret value). We have a test
+// to confirm that.
+INTERCEPTOR(void, mallinfo, __sanitizer_mallinfo *sret) {
+#ifdef __aarch64__
+ uptr r8;
+ asm volatile("mov %0,x8" : "=r" (r8));
+ sret = reinterpret_cast<__sanitizer_mallinfo*>(r8);
+#endif
+ REAL(memset)(sret, 0, sizeof(*sret));
+}
+#define HWASAN_MAYBE_INTERCEPT_MALLINFO INTERCEPT_FUNCTION(mallinfo)
+#else
+#define HWASAN_MAYBE_INTERCEPT_MALLINFO
+#endif
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR(int, mallopt, int cmd, int value) {
+ return -1;
+}
+#define HWASAN_MAYBE_INTERCEPT_MALLOPT INTERCEPT_FUNCTION(mallopt)
+#else
+#define HWASAN_MAYBE_INTERCEPT_MALLOPT
+#endif
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR(void, malloc_stats, void) {
+ // FIXME: implement, but don't call REAL(malloc_stats)!
+}
+#define HWASAN_MAYBE_INTERCEPT_MALLOC_STATS INTERCEPT_FUNCTION(malloc_stats)
+#else
+#define HWASAN_MAYBE_INTERCEPT_MALLOC_STATS
+#endif
+
+
+INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) {
+ GET_MALLOC_STACK_TRACE;
+ if (UNLIKELY(!hwasan_inited))
+ // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
+ return AllocateFromLocalPool(nmemb * size);
+ return hwasan_calloc(nmemb, size, &stack);
+}
+
+INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) {
+ GET_MALLOC_STACK_TRACE;
+ if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
+ uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
+ uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
+ void *new_ptr;
+ if (UNLIKELY(!hwasan_inited)) {
+ new_ptr = AllocateFromLocalPool(copy_size);
+ } else {
+ copy_size = size;
+ new_ptr = hwasan_malloc(copy_size, &stack);
+ }
+ internal_memcpy(new_ptr, ptr, copy_size);
+ return new_ptr;
+ }
+ return hwasan_realloc(ptr, size, &stack);
+}
+
+INTERCEPTOR(void *, malloc, SIZE_T size) {
+ GET_MALLOC_STACK_TRACE;
+ if (UNLIKELY(!hwasan_inited))
+ // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
+ return AllocateFromLocalPool(size);
+ return hwasan_malloc(size, &stack);
+}
+
+
+INTERCEPTOR(void *, mmap, void *addr, SIZE_T length, int prot, int flags,
+ int fd, OFF_T offset) {
+ if (hwasan_init_is_running)
+ return REAL(mmap)(addr, length, prot, flags, fd, offset);
+ ENSURE_HWASAN_INITED();
+ if (addr && !MEM_IS_APP(addr)) {
+ if (flags & map_fixed) {
+ errno = errno_EINVAL;
+ return (void *)-1;
+ } else {
+ addr = nullptr;
+ }
+ }
+ void *res = REAL(mmap)(addr, length, prot, flags, fd, offset);
+ return res;
+}
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR(void *, mmap64, void *addr, SIZE_T length, int prot, int flags,
+ int fd, OFF64_T offset) {
+ ENSURE_HWASAN_INITED();
+ if (addr && !MEM_IS_APP(addr)) {
+ if (flags & map_fixed) {
+ errno = errno_EINVAL;
+ return (void *)-1;
+ } else {
+ addr = nullptr;
+ }
+ }
+ void *res = REAL(mmap64)(addr, length, prot, flags, fd, offset);
+ return res;
+}
+#define HWASAN_MAYBE_INTERCEPT_MMAP64 INTERCEPT_FUNCTION(mmap64)
+#else
+#define HWASAN_MAYBE_INTERCEPT_MMAP64
+#endif
+
+extern "C" int pthread_attr_init(void *attr);
+extern "C" int pthread_attr_destroy(void *attr);
+
+static void *HwasanThreadStartFunc(void *arg) {
+ HwasanThread *t = (HwasanThread *)arg;
+ SetCurrentThread(t);
+ return t->ThreadStart();
+}
+
+INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*),
+ void * param) {
+ ENSURE_HWASAN_INITED(); // for GetTlsSize()
+ __sanitizer_pthread_attr_t myattr;
+ if (!attr) {
+ pthread_attr_init(&myattr);
+ attr = &myattr;
+ }
+
+ AdjustStackSize(attr);
+
+ HwasanThread *t = HwasanThread::Create(callback, param);
+
+ int res = REAL(pthread_create)(th, attr, HwasanThreadStartFunc, t);
+
+ if (attr == &myattr)
+ pthread_attr_destroy(&myattr);
+ return res;
+}
+
+static void BeforeFork() {
+ StackDepotLockAll();
+}
+
+static void AfterFork() {
+ StackDepotUnlockAll();
+}
+
+INTERCEPTOR(int, fork, void) {
+ ENSURE_HWASAN_INITED();
+ BeforeFork();
+ int pid = REAL(fork)();
+ AfterFork();
+ return pid;
+}
+
+
+struct HwasanInterceptorContext {
+ bool in_interceptor_scope;
+};
+
+namespace __hwasan {
+
+int OnExit() {
+ // FIXME: ask frontend whether we need to return failure.
+ return 0;
+}
+
+} // namespace __hwasan
+
+// A version of CHECK_UNPOISONED using a saved scope value. Used in common
+// interceptors.
+#define CHECK_UNPOISONED_CTX(ctx, x, n) \
+ do { \
+ if (!((HwasanInterceptorContext *)ctx)->in_interceptor_scope) \
+ CHECK_UNPOISONED_0(x, n); \
+ } while (0)
+
+#define HWASAN_INTERCEPT_FUNC(name) \
+ do { \
+ if ((!INTERCEPT_FUNCTION(name) || !REAL(name))) \
+ VReport(1, "HWAddressSanitizer: failed to intercept '" #name "'\n"); \
+ } while (0)
+
+#define HWASAN_INTERCEPT_FUNC_VER(name, ver) \
+ do { \
+ if ((!INTERCEPT_FUNCTION_VER(name, ver) || !REAL(name))) \
+ VReport( \
+ 1, "HWAddressSanitizer: failed to intercept '" #name "@@" #ver "'\n"); \
+ } while (0)
+
+#define COMMON_INTERCEPT_FUNCTION(name) HWASAN_INTERCEPT_FUNC(name)
+#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \
+ HWASAN_INTERCEPT_FUNC_VER(name, ver)
+#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \
+ CHECK_UNPOISONED_CTX(ctx, ptr, size)
+#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \
+ CHECK_UNPOISONED_CTX(ctx, ptr, size)
+#define COMMON_INTERCEPTOR_INITIALIZE_RANGE(ptr, size) \
+ HWASAN_WRITE_RANGE(ctx, ptr, size)
+#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \
+ if (hwasan_init_is_running) return REAL(func)(__VA_ARGS__); \
+ ENSURE_HWASAN_INITED(); \
+ HwasanInterceptorContext hwasan_ctx = {IsInInterceptorScope()}; \
+ ctx = (void *)&hwasan_ctx; \
+ (void)ctx; \
+ InterceptorScope interceptor_scope;
+#define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \
+ do { \
+ } while (false)
+#define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \
+ do { \
+ } while (false)
+#define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \
+ do { \
+ } while (false)
+#define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \
+ do { \
+ } while (false)
+#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \
+ do { \
+ } while (false) // FIXME
+#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \
+ do { \
+ } while (false) // FIXME
+#define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name)
+#define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit()
+
+#define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \
+ if (HwasanThread *t = GetCurrentThread()) { \
+ *begin = t->tls_begin(); \
+ *end = t->tls_end(); \
+ } else { \
+ *begin = *end = 0; \
+ }
+
+#include "sanitizer_common/sanitizer_platform_interceptors.h"
+#include "sanitizer_common/sanitizer_common_interceptors.inc"
+#include "sanitizer_common/sanitizer_signal_interceptors.inc"
+
+#define COMMON_SYSCALL_PRE_READ_RANGE(p, s) CHECK_UNPOISONED(p, s)
+#define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \
+ do { \
+ (void)(p); \
+ (void)(s); \
+ } while (false)
+#define COMMON_SYSCALL_POST_READ_RANGE(p, s) \
+ do { \
+ (void)(p); \
+ (void)(s); \
+ } while (false)
+#define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \
+ do { \
+ (void)(p); \
+ (void)(s); \
+ } while (false)
+#include "sanitizer_common/sanitizer_common_syscalls.inc"
+
+
+
+namespace __hwasan {
+
+void InitializeInterceptors() {
+ static int inited = 0;
+ CHECK_EQ(inited, 0);
+ InitializeCommonInterceptors();
+ InitializeSignalInterceptors();
+
+ INTERCEPT_FUNCTION(mmap);
+ HWASAN_MAYBE_INTERCEPT_MMAP64;
+ INTERCEPT_FUNCTION(posix_memalign);
+ HWASAN_MAYBE_INTERCEPT_MEMALIGN;
+ INTERCEPT_FUNCTION(__libc_memalign);
+ INTERCEPT_FUNCTION(valloc);
+ HWASAN_MAYBE_INTERCEPT_PVALLOC;
+ INTERCEPT_FUNCTION(malloc);
+ INTERCEPT_FUNCTION(calloc);
+ INTERCEPT_FUNCTION(realloc);
+ INTERCEPT_FUNCTION(free);
+ HWASAN_MAYBE_INTERCEPT_CFREE;
+ INTERCEPT_FUNCTION(malloc_usable_size);
+ HWASAN_MAYBE_INTERCEPT_MALLINFO;
+ HWASAN_MAYBE_INTERCEPT_MALLOPT;
+ HWASAN_MAYBE_INTERCEPT_MALLOC_STATS;
+ INTERCEPT_FUNCTION(pthread_create);
+ INTERCEPT_FUNCTION(fork);
+
+ inited = 1;
+}
+} // namespace __hwasan
diff --git a/lib/hwasan/hwasan_interface_internal.h b/lib/hwasan/hwasan_interface_internal.h
new file mode 100644
index 000000000..07f73ce22
--- /dev/null
+++ b/lib/hwasan/hwasan_interface_internal.h
@@ -0,0 +1,105 @@
+//===-- hwasan_interface_internal.h -------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Private Hwasan interface header.
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_INTERFACE_INTERNAL_H
+#define HWASAN_INTERFACE_INTERNAL_H
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_init();
+
+using __sanitizer::uptr;
+using __sanitizer::sptr;
+using __sanitizer::uu64;
+using __sanitizer::uu32;
+using __sanitizer::uu16;
+using __sanitizer::u64;
+using __sanitizer::u32;
+using __sanitizer::u16;
+using __sanitizer::u8;
+
+SANITIZER_INTERFACE_ATTRIBUTE
+extern uptr __hwasan_shadow_memory_dynamic_address;
+
+// Hidden alias for internal access.
+__attribute__((visibility("hidden")))
+extern uptr __hwasan_shadow_memory_dynamic_address_internal;
+
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load(uptr, uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load1(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load2(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load4(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load8(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load16(uptr);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store(uptr, uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store1(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store2(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store4(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store8(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store16(uptr);
+
+// Returns the offset of the first tag mismatch or -1 if the whole range is
+// good.
+SANITIZER_INTERFACE_ATTRIBUTE
+sptr __hwasan_test_shadow(const void *x, uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+/* OPTIONAL */ const char* __hwasan_default_options();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_print_shadow(const void *x, uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+u16 __sanitizer_unaligned_load16(const uu16 *p);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+u32 __sanitizer_unaligned_load32(const uu32 *p);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+u64 __sanitizer_unaligned_load64(const uu64 *p);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_unaligned_store16(uu16 *p, u16 x);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_unaligned_store32(uu32 *p, u32 x);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_unaligned_store64(uu64 *p, u64 x);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_enable_allocator_tagging();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_disable_allocator_tagging();
+
+} // extern "C"
+
+#endif // HWASAN_INTERFACE_INTERNAL_H
diff --git a/lib/hwasan/hwasan_linux.cc b/lib/hwasan/hwasan_linux.cc
new file mode 100644
index 000000000..c07f9928a
--- /dev/null
+++ b/lib/hwasan/hwasan_linux.cc
@@ -0,0 +1,194 @@
+//===-- hwasan_linux.cc -----------------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Linux-, NetBSD- and FreeBSD-specific code.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
+
+#include "hwasan.h"
+#include "hwasan_thread.h"
+
+#include <elf.h>
+#include <link.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <signal.h>
+#include <unistd.h>
+#include <unwind.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_procmaps.h"
+
+uptr __hwasan_shadow_memory_dynamic_address;
+
+__attribute__((alias("__hwasan_shadow_memory_dynamic_address")))
+extern uptr __hwasan_shadow_memory_dynamic_address_internal;
+
+namespace __hwasan {
+
+bool InitShadow() {
+ const uptr maxVirtualAddress = GetMaxUserVirtualAddress();
+ uptr shadow_size = MEM_TO_SHADOW_OFFSET(maxVirtualAddress) + 1;
+ __hwasan_shadow_memory_dynamic_address =
+ reinterpret_cast<uptr>(MmapNoReserveOrDie(shadow_size, "shadow"));
+ return true;
+}
+
+static void HwasanAtExit(void) {
+ if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0))
+ ReportStats();
+ if (hwasan_report_count > 0) {
+ // ReportAtExitStatistics();
+ if (common_flags()->exitcode)
+ internal__exit(common_flags()->exitcode);
+ }
+}
+
+void InstallAtExitHandler() {
+ atexit(HwasanAtExit);
+}
+
+// ---------------------- TSD ---------------- {{{1
+
+static pthread_key_t tsd_key;
+static bool tsd_key_inited = false;
+
+void HwasanTSDInit(void (*destructor)(void *tsd)) {
+ CHECK(!tsd_key_inited);
+ tsd_key_inited = true;
+ CHECK_EQ(0, pthread_key_create(&tsd_key, destructor));
+}
+
+HwasanThread *GetCurrentThread() {
+ return (HwasanThread*)pthread_getspecific(tsd_key);
+}
+
+void SetCurrentThread(HwasanThread *t) {
+ // Make sure that HwasanTSDDtor gets called at the end.
+ CHECK(tsd_key_inited);
+ // Make sure we do not reset the current HwasanThread.
+ CHECK_EQ(0, pthread_getspecific(tsd_key));
+ pthread_setspecific(tsd_key, (void *)t);
+}
+
+void HwasanTSDDtor(void *tsd) {
+ HwasanThread *t = (HwasanThread*)tsd;
+ if (t->destructor_iterations_ > 1) {
+ t->destructor_iterations_--;
+ CHECK_EQ(0, pthread_setspecific(tsd_key, tsd));
+ return;
+ }
+ // Make sure that signal handler can not see a stale current thread pointer.
+ atomic_signal_fence(memory_order_seq_cst);
+ HwasanThread::TSDDtor(tsd);
+}
+
+struct AccessInfo {
+ uptr addr;
+ uptr size;
+ bool is_store;
+ bool is_load;
+};
+
+#if defined(__aarch64__)
+static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
+ AccessInfo ai;
+ uptr pc = (uptr)info->si_addr;
+
+ struct {
+ uptr addr;
+ unsigned size;
+ bool is_store;
+ } handlers[] = {
+ {(uptr)&__hwasan_load1, 1, false}, {(uptr)&__hwasan_load2, 2, false},
+ {(uptr)&__hwasan_load4, 4, false}, {(uptr)&__hwasan_load8, 8, false},
+ {(uptr)&__hwasan_load16, 16, false}, {(uptr)&__hwasan_load, 0, false},
+ {(uptr)&__hwasan_store1, 1, true}, {(uptr)&__hwasan_store2, 2, true},
+ {(uptr)&__hwasan_store4, 4, true}, {(uptr)&__hwasan_store8, 8, true},
+ {(uptr)&__hwasan_store16, 16, true}, {(uptr)&__hwasan_store, 0, true}};
+ int best = -1;
+ uptr best_distance = 0;
+ for (size_t i = 0; i < sizeof(handlers) / sizeof(handlers[0]); ++i) {
+ uptr handler = handlers[i].addr;
+ // Don't accept pc == handler: HLT is never the first instruction.
+ if (pc <= handler) continue;
+ uptr distance = pc - handler;
+ if (distance > 256) continue;
+ if (best == -1 || best_distance > distance) {
+ best = i;
+ best_distance = distance;
+ }
+ }
+
+ // Not ours.
+ if (best == -1)
+ return AccessInfo{0, 0, false, false};
+
+ ai.is_store = handlers[best].is_store;
+ ai.is_load = !handlers[best].is_store;
+ ai.size = handlers[best].size;
+
+ ai.addr = uc->uc_mcontext.regs[0];
+ if (ai.size == 0)
+ ai.size = uc->uc_mcontext.regs[1];
+ return ai;
+}
+#else
+static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
+ return AccessInfo{0, 0, false, false};
+}
+#endif
+
+static void HwasanOnSIGILL(int signo, siginfo_t *info, ucontext_t *uc) {
+ SignalContext sig{info, uc};
+ AccessInfo ai = GetAccessInfo(info, uc);
+ if (!ai.is_store && !ai.is_load)
+ return;
+
+ InternalScopedBuffer<BufferedStackTrace> stack_buffer(1);
+ BufferedStackTrace *stack = stack_buffer.data();
+ stack->Reset();
+ GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, uc,
+ common_flags()->fast_unwind_on_fatal);
+
+ ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store);
+
+ ++hwasan_report_count;
+ if (flags()->halt_on_error)
+ Die();
+ else
+ uc->uc_mcontext.pc += 4;
+}
+
+static void OnStackUnwind(const SignalContext &sig, const void *,
+ BufferedStackTrace *stack) {
+ GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context,
+ common_flags()->fast_unwind_on_fatal);
+}
+
+void HwasanOnDeadlySignal(int signo, void *info, void *context) {
+ // Probably a tag mismatch.
+ // FIXME: detect pc range in __hwasan_load* or __hwasan_store*.
+ if (signo == SIGILL)
+ HwasanOnSIGILL(signo, (siginfo_t *)info, (ucontext_t*)context);
+ else
+ HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr);
+}
+
+
+} // namespace __hwasan
+
+#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
diff --git a/lib/hwasan/hwasan_new_delete.cc b/lib/hwasan/hwasan_new_delete.cc
new file mode 100644
index 000000000..3ccc26734
--- /dev/null
+++ b/lib/hwasan/hwasan_new_delete.cc
@@ -0,0 +1,66 @@
+//===-- hwasan_new_delete.cc ------------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Interceptors for operators new and delete.
+//===----------------------------------------------------------------------===//
+
+#include "hwasan.h"
+#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator.h"
+
+#if HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE
+
+#include <stddef.h>
+
+using namespace __hwasan; // NOLINT
+
+// Fake std::nothrow_t to avoid including <new>.
+namespace std {
+ struct nothrow_t {};
+} // namespace std
+
+
+// TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
+#define OPERATOR_NEW_BODY(nothrow) \
+ GET_MALLOC_STACK_TRACE; \
+ void *res = hwasan_malloc(size, &stack);\
+ if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();\
+ return res
+
+INTERCEPTOR_ATTRIBUTE
+void *operator new(size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); }
+INTERCEPTOR_ATTRIBUTE
+void *operator new[](size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); }
+INTERCEPTOR_ATTRIBUTE
+void *operator new(size_t size, std::nothrow_t const&) {
+ OPERATOR_NEW_BODY(true /*nothrow*/);
+}
+INTERCEPTOR_ATTRIBUTE
+void *operator new[](size_t size, std::nothrow_t const&) {
+ OPERATOR_NEW_BODY(true /*nothrow*/);
+}
+
+#define OPERATOR_DELETE_BODY \
+ GET_MALLOC_STACK_TRACE; \
+ if (ptr) HwasanDeallocate(&stack, ptr)
+
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr, std::nothrow_t const&) {
+ OPERATOR_DELETE_BODY;
+}
+
+#endif // HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE
diff --git a/lib/hwasan/hwasan_poisoning.cc b/lib/hwasan/hwasan_poisoning.cc
new file mode 100644
index 000000000..411fd05b1
--- /dev/null
+++ b/lib/hwasan/hwasan_poisoning.cc
@@ -0,0 +1,36 @@
+//===-- hwasan_poisoning.cc ---------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "hwasan_poisoning.h"
+
+#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_common.h"
+
+namespace __hwasan {
+
+uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) {
+ CHECK(IsAligned(p, kShadowAlignment));
+ CHECK(IsAligned(size, kShadowAlignment));
+ uptr shadow_start = MEM_TO_SHADOW(p);
+ uptr shadow_size = MEM_TO_SHADOW_OFFSET(size);
+ internal_memset((void *)shadow_start, tag, shadow_size);
+ return AddTagToPointer(p, tag);
+}
+
+uptr TagMemory(uptr p, uptr size, tag_t tag) {
+ uptr start = RoundDownTo(p, kShadowAlignment);
+ uptr end = RoundUpTo(p + size, kShadowAlignment);
+ return TagMemoryAligned(start, end - start, tag);
+}
+
+} // namespace __hwasan
diff --git a/lib/hwasan/hwasan_poisoning.h b/lib/hwasan/hwasan_poisoning.h
new file mode 100644
index 000000000..b44a91f97
--- /dev/null
+++ b/lib/hwasan/hwasan_poisoning.h
@@ -0,0 +1,25 @@
+//===-- hwasan_poisoning.h ----------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_POISONING_H
+#define HWASAN_POISONING_H
+
+#include "hwasan.h"
+
+namespace __hwasan {
+uptr TagMemory(uptr p, uptr size, tag_t tag);
+uptr TagMemoryAligned(uptr p, uptr size, tag_t tag);
+
+} // namespace __hwasan
+
+#endif // HWASAN_POISONING_H
diff --git a/lib/hwasan/hwasan_report.cc b/lib/hwasan/hwasan_report.cc
new file mode 100644
index 000000000..a3c670949
--- /dev/null
+++ b/lib/hwasan/hwasan_report.cc
@@ -0,0 +1,133 @@
+//===-- hwasan_report.cc ----------------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Error reporting.
+//===----------------------------------------------------------------------===//
+
+#include "hwasan.h"
+#include "hwasan_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_internal.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_mutex.h"
+#include "sanitizer_common/sanitizer_report_decorator.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_symbolizer.h"
+
+using namespace __sanitizer;
+
+namespace __hwasan {
+
+static StackTrace GetStackTraceFromId(u32 id) {
+ CHECK(id);
+ StackTrace res = StackDepotGet(id);
+ CHECK(res.trace);
+ return res;
+}
+
+class Decorator: public __sanitizer::SanitizerCommonDecorator {
+ public:
+ Decorator() : SanitizerCommonDecorator() { }
+ const char *Allocation() { return Magenta(); }
+ const char *Origin() { return Magenta(); }
+ const char *Name() { return Green(); }
+};
+
+struct HeapAddressDescription {
+ uptr addr;
+ u32 alloc_stack_id;
+ u32 free_stack_id;
+
+ void Print() const {
+ Decorator d;
+ if (free_stack_id) {
+ Printf("%sfreed here:%s\n", d.Allocation(), d.Default());
+ GetStackTraceFromId(free_stack_id).Print();
+ Printf("%spreviously allocated here:%s\n", d.Allocation(), d.Default());
+ } else {
+ Printf("%sallocated here:%s\n", d.Allocation(), d.Default());
+ }
+ GetStackTraceFromId(alloc_stack_id).Print();
+ }
+};
+
+bool GetHeapAddressInformation(uptr addr, uptr access_size,
+ HeapAddressDescription *description) {
+ HwasanChunkView chunk = FindHeapChunkByAddress(addr);
+ if (!chunk.IsValid())
+ return false;
+ description->addr = addr;
+ description->alloc_stack_id = chunk.GetAllocStackId();
+ description->free_stack_id = chunk.GetFreeStackId();
+ return true;
+}
+
+void PrintAddressDescription(uptr addr, uptr access_size) {
+ HeapAddressDescription heap_description;
+ if (GetHeapAddressInformation(addr, access_size, &heap_description)) {
+ heap_description.Print();
+ return;
+ }
+ // We exhausted our possibilities. Bail out.
+ Printf("HWAddressSanitizer can not describe address in more detail.\n");
+}
+
+void ReportInvalidAccess(StackTrace *stack, u32 origin) {
+ ScopedErrorReportLock l;
+
+ Decorator d;
+ Printf("%s", d.Warning());
+ Report("WARNING: HWAddressSanitizer: invalid access\n");
+ Printf("%s", d.Default());
+ stack->Print();
+ ReportErrorSummary("invalid-access", stack);
+}
+
+void ReportStats() {}
+
+void ReportInvalidAccessInsideAddressRange(const char *what, const void *start,
+ uptr size, uptr offset) {
+ ScopedErrorReportLock l;
+
+ Decorator d;
+ Printf("%s", d.Warning());
+ Printf("%sTag mismatch in %s%s%s at offset %zu inside [%p, %zu)%s\n",
+ d.Warning(), d.Name(), what, d.Warning(), offset, start, size,
+ d.Default());
+ PrintAddressDescription((uptr)start + offset, 1);
+ // if (__sanitizer::Verbosity())
+ // DescribeMemoryRange(start, size);
+}
+
+void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size,
+ bool is_store) {
+ ScopedErrorReportLock l;
+
+ Decorator d;
+ Printf("%s", d.Warning());
+ uptr address = GetAddressFromPointer(addr);
+ Printf("%s of size %zu at %p\n", is_store ? "WRITE" : "READ", access_size,
+ address);
+
+ tag_t ptr_tag = GetTagFromPointer(addr);
+ tag_t mem_tag = *(tag_t *)MEM_TO_SHADOW(address);
+ Printf("pointer tag 0x%x\nmemory tag 0x%x\n", ptr_tag, mem_tag);
+ Printf("%s", d.Default());
+
+ stack->Print();
+
+ PrintAddressDescription(address, access_size);
+
+ ReportErrorSummary("tag-mismatch", stack);
+}
+
+
+} // namespace __hwasan
diff --git a/lib/hwasan/hwasan_thread.cc b/lib/hwasan/hwasan_thread.cc
new file mode 100644
index 000000000..d6551ffc6
--- /dev/null
+++ b/lib/hwasan/hwasan_thread.cc
@@ -0,0 +1,75 @@
+
+#include "hwasan.h"
+#include "hwasan_thread.h"
+#include "hwasan_poisoning.h"
+#include "hwasan_interface_internal.h"
+
+#include "sanitizer_common/sanitizer_tls_get_addr.h"
+
+namespace __hwasan {
+
+HwasanThread *HwasanThread::Create(thread_callback_t start_routine,
+ void *arg) {
+ uptr PageSize = GetPageSizeCached();
+ uptr size = RoundUpTo(sizeof(HwasanThread), PageSize);
+ HwasanThread *thread = (HwasanThread*)MmapOrDie(size, __func__);
+ thread->start_routine_ = start_routine;
+ thread->arg_ = arg;
+ thread->destructor_iterations_ = GetPthreadDestructorIterations();
+
+ return thread;
+}
+
+void HwasanThread::SetThreadStackAndTls() {
+ uptr tls_size = 0;
+ uptr stack_size = 0;
+ GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size,
+ &tls_begin_, &tls_size);
+ stack_top_ = stack_bottom_ + stack_size;
+ tls_end_ = tls_begin_ + tls_size;
+
+ int local;
+ CHECK(AddrIsInStack((uptr)&local));
+}
+
+void HwasanThread::Init() {
+ SetThreadStackAndTls();
+ CHECK(MEM_IS_APP(stack_bottom_));
+ CHECK(MEM_IS_APP(stack_top_ - 1));
+}
+
+void HwasanThread::TSDDtor(void *tsd) {
+ HwasanThread *t = (HwasanThread*)tsd;
+ t->Destroy();
+}
+
+void HwasanThread::ClearShadowForThreadStackAndTLS() {
+ TagMemory(stack_bottom_, stack_top_ - stack_bottom_, 0);
+ if (tls_begin_ != tls_end_)
+ TagMemory(tls_begin_, tls_end_ - tls_begin_, 0);
+}
+
+void HwasanThread::Destroy() {
+ malloc_storage().CommitBack();
+ ClearShadowForThreadStackAndTLS();
+ uptr size = RoundUpTo(sizeof(HwasanThread), GetPageSizeCached());
+ UnmapOrDie(this, size);
+ DTLS_Destroy();
+}
+
+thread_return_t HwasanThread::ThreadStart() {
+ Init();
+
+ if (!start_routine_) {
+ // start_routine_ == 0 if we're on the main thread or on one of the
+ // OS X libdispatch worker threads. But nobody is supposed to call
+ // ThreadStart() for the worker threads.
+ return 0;
+ }
+
+ thread_return_t res = start_routine_(arg_);
+
+ return res;
+}
+
+} // namespace __hwasan
diff --git a/lib/hwasan/hwasan_thread.h b/lib/hwasan/hwasan_thread.h
new file mode 100644
index 000000000..96f1bb813
--- /dev/null
+++ b/lib/hwasan/hwasan_thread.h
@@ -0,0 +1,81 @@
+//===-- hwasan_thread.h -------------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_THREAD_H
+#define HWASAN_THREAD_H
+
+#include "hwasan_allocator.h"
+#include "sanitizer_common/sanitizer_common.h"
+
+namespace __hwasan {
+
+class HwasanThread {
+ public:
+ static HwasanThread *Create(thread_callback_t start_routine, void *arg);
+ static void TSDDtor(void *tsd);
+ void Destroy();
+
+ void Init(); // Should be called from the thread itself.
+ thread_return_t ThreadStart();
+
+ uptr stack_top() { return stack_top_; }
+ uptr stack_bottom() { return stack_bottom_; }
+ uptr tls_begin() { return tls_begin_; }
+ uptr tls_end() { return tls_end_; }
+ bool IsMainThread() { return start_routine_ == nullptr; }
+
+ bool AddrIsInStack(uptr addr) {
+ return addr >= stack_bottom_ && addr < stack_top_;
+ }
+
+ bool InSignalHandler() { return in_signal_handler_; }
+ void EnterSignalHandler() { in_signal_handler_++; }
+ void LeaveSignalHandler() { in_signal_handler_--; }
+
+ bool InSymbolizer() { return in_symbolizer_; }
+ void EnterSymbolizer() { in_symbolizer_++; }
+ void LeaveSymbolizer() { in_symbolizer_--; }
+
+ bool InInterceptorScope() { return in_interceptor_scope_; }
+ void EnterInterceptorScope() { in_interceptor_scope_++; }
+ void LeaveInterceptorScope() { in_interceptor_scope_--; }
+
+ HwasanThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; }
+
+ int destructor_iterations_;
+
+ private:
+ // NOTE: There is no HwasanThread constructor. It is allocated
+ // via mmap() and *must* be valid in zero-initialized state.
+ void SetThreadStackAndTls();
+ void ClearShadowForThreadStackAndTLS();
+ thread_callback_t start_routine_;
+ void *arg_;
+ uptr stack_top_;
+ uptr stack_bottom_;
+ uptr tls_begin_;
+ uptr tls_end_;
+
+ unsigned in_signal_handler_;
+ unsigned in_symbolizer_;
+ unsigned in_interceptor_scope_;
+
+ HwasanThreadLocalMallocStorage malloc_storage_;
+};
+
+HwasanThread *GetCurrentThread();
+void SetCurrentThread(HwasanThread *t);
+
+} // namespace __hwasan
+
+#endif // HWASAN_THREAD_H
diff --git a/lib/sanitizer_common/sanitizer_internal_defs.h b/lib/sanitizer_common/sanitizer_internal_defs.h
index dcc8db414..e5da5bd15 100644
--- a/lib/sanitizer_common/sanitizer_internal_defs.h
+++ b/lib/sanitizer_common/sanitizer_internal_defs.h
@@ -390,6 +390,7 @@ namespace __dfsan { using namespace __sanitizer; } // NOLINT
namespace __esan { using namespace __sanitizer; } // NOLINT
namespace __lsan { using namespace __sanitizer; } // NOLINT
namespace __msan { using namespace __sanitizer; } // NOLINT
+namespace __hwasan { using namespace __sanitizer; } // NOLINT
namespace __tsan { using namespace __sanitizer; } // NOLINT
namespace __scudo { using namespace __sanitizer; } // NOLINT
namespace __ubsan { using namespace __sanitizer; } // NOLINT