diff options
Diffstat (limited to 'libsanitizer/tsan')
48 files changed, 3948 insertions, 1045 deletions
diff --git a/libsanitizer/tsan/Makefile.am b/libsanitizer/tsan/Makefile.am index 9833de20bd82..7a0c29711433 100644 --- a/libsanitizer/tsan/Makefile.am +++ b/libsanitizer/tsan/Makefile.am @@ -10,13 +10,16 @@ AM_CXXFLAGS += -std=gnu++11 ACLOCAL_AMFLAGS = -I m4 toolexeclib_LTLIBRARIES = libtsan.la +nodist_toolexeclib_HEADERS = libtsan_preinit.o tsan_files = \ tsan_clock.cc \ + tsan_debugging.cc \ tsan_fd.cc \ tsan_flags.cc \ tsan_ignoreset.cc \ tsan_interceptors.cc \ + tsan_interceptors_mac.cc \ tsan_interface_ann.cc \ tsan_interface_atomic.cc \ tsan_interface.cc \ @@ -35,6 +38,7 @@ tsan_files = \ tsan_report.cc \ tsan_rtl.cc \ tsan_rtl_mutex.cc \ + tsan_rtl_proc.cc \ tsan_rtl_report.cc \ tsan_rtl_thread.cc \ tsan_stack_trace.cc \ @@ -44,7 +48,7 @@ tsan_files = \ tsan_sync.cc libtsan_la_SOURCES = $(tsan_files) -EXTRA_libtsan_la_SOURCES = tsan_rtl_amd64.S tsan_rtl_aarch64.S +EXTRA_libtsan_la_SOURCES = tsan_rtl_amd64.S tsan_rtl_aarch64.S tsan_rtl_mips64.S tsan_rtl_ppc64.S libtsan_la_LIBADD = $(top_builddir)/sanitizer_common/libsanitizer_common.la $(top_builddir)/interception/libinterception.la $(TSAN_TARGET_DEPENDENT_OBJECTS) libtsan_la_DEPENDENCIES = $(top_builddir)/sanitizer_common/libsanitizer_common.la $(top_builddir)/interception/libinterception.la $(TSAN_TARGET_DEPENDENT_OBJECTS) if LIBBACKTRACE_SUPPORTED @@ -54,6 +58,9 @@ endif libtsan_la_LIBADD += $(LIBSTDCXX_RAW_CXX_LDFLAGS) libtsan_la_LDFLAGS = -version-info `grep -v '^\#' $(srcdir)/libtool-version` $(link_libtsan) +libtsan_preinit.o: tsan_preinit.o + cp $< $@ + # Work around what appears to be a GNU make bug handling MAKEFLAGS # values defined in terms of make variables, as is the case for CC and # friends when we are called from the top level Makefile. diff --git a/libsanitizer/tsan/Makefile.in b/libsanitizer/tsan/Makefile.in index de8de7c66742..4d4ec47becc2 100644 --- a/libsanitizer/tsan/Makefile.in +++ b/libsanitizer/tsan/Makefile.in @@ -15,6 +15,7 @@ @SET_MAKE@ + VPATH = @srcdir@ am__make_dryrun = \ { \ @@ -101,20 +102,22 @@ am__uninstall_files_from_dir = { \ || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ $(am__cd) "$$dir" && rm -f $$files; }; \ } -am__installdirs = "$(DESTDIR)$(toolexeclibdir)" +am__installdirs = "$(DESTDIR)$(toolexeclibdir)" \ + "$(DESTDIR)$(toolexeclibdir)" LTLIBRARIES = $(toolexeclib_LTLIBRARIES) am__DEPENDENCIES_1 = -am__objects_1 = tsan_clock.lo tsan_fd.lo tsan_flags.lo \ - tsan_ignoreset.lo tsan_interceptors.lo tsan_interface_ann.lo \ +am__objects_1 = tsan_clock.lo tsan_debugging.lo tsan_fd.lo \ + tsan_flags.lo tsan_ignoreset.lo tsan_interceptors.lo \ + tsan_interceptors_mac.lo tsan_interface_ann.lo \ tsan_interface_atomic.lo tsan_interface.lo \ tsan_interface_java.lo tsan_libdispatch_mac.lo \ tsan_malloc_mac.lo tsan_md5.lo tsan_mman.lo tsan_mutex.lo \ tsan_mutexset.lo tsan_new_delete.lo tsan_platform_linux.lo \ tsan_platform_mac.lo tsan_platform_posix.lo \ tsan_platform_windows.lo tsan_report.lo tsan_rtl.lo \ - tsan_rtl_mutex.lo tsan_rtl_report.lo tsan_rtl_thread.lo \ - tsan_stack_trace.lo tsan_stat.lo tsan_suppressions.lo \ - tsan_symbolize.lo tsan_sync.lo + tsan_rtl_mutex.lo tsan_rtl_proc.lo tsan_rtl_report.lo \ + tsan_rtl_thread.lo tsan_stack_trace.lo tsan_stat.lo \ + tsan_suppressions.lo tsan_symbolize.lo tsan_sync.lo am_libtsan_la_OBJECTS = $(am__objects_1) libtsan_la_OBJECTS = $(am_libtsan_la_OBJECTS) libtsan_la_LINK = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) \ @@ -153,6 +156,7 @@ am__can_run_installinfo = \ n|no|NO) false;; \ *) (install-info --version) >/dev/null 2>&1;; \ esac +HEADERS = $(nodist_toolexeclib_HEADERS) ETAGS = etags CTAGS = ctags ACLOCAL = @ACLOCAL@ @@ -225,6 +229,7 @@ PACKAGE_VERSION = @PACKAGE_VERSION@ PATH_SEPARATOR = @PATH_SEPARATOR@ RANLIB = @RANLIB@ RPC_DEFS = @RPC_DEFS@ +SANITIZER_COMMON_TARGET_DEPENDENT_OBJECTS = @SANITIZER_COMMON_TARGET_DEPENDENT_OBJECTS@ SED = @SED@ SET_MAKE = @SET_MAKE@ SHELL = @SHELL@ @@ -309,12 +314,15 @@ AM_CXXFLAGS = -Wall -W -Wno-unused-parameter -Wwrite-strings -pedantic \ -std=gnu++11 ACLOCAL_AMFLAGS = -I m4 toolexeclib_LTLIBRARIES = libtsan.la +nodist_toolexeclib_HEADERS = libtsan_preinit.o tsan_files = \ tsan_clock.cc \ + tsan_debugging.cc \ tsan_fd.cc \ tsan_flags.cc \ tsan_ignoreset.cc \ tsan_interceptors.cc \ + tsan_interceptors_mac.cc \ tsan_interface_ann.cc \ tsan_interface_atomic.cc \ tsan_interface.cc \ @@ -333,6 +341,7 @@ tsan_files = \ tsan_report.cc \ tsan_rtl.cc \ tsan_rtl_mutex.cc \ + tsan_rtl_proc.cc \ tsan_rtl_report.cc \ tsan_rtl_thread.cc \ tsan_stack_trace.cc \ @@ -342,7 +351,7 @@ tsan_files = \ tsan_sync.cc libtsan_la_SOURCES = $(tsan_files) -EXTRA_libtsan_la_SOURCES = tsan_rtl_amd64.S tsan_rtl_aarch64.S +EXTRA_libtsan_la_SOURCES = tsan_rtl_amd64.S tsan_rtl_aarch64.S tsan_rtl_mips64.S tsan_rtl_ppc64.S libtsan_la_LIBADD = \ $(top_builddir)/sanitizer_common/libsanitizer_common.la \ $(top_builddir)/interception/libinterception.la \ @@ -469,10 +478,12 @@ distclean-compile: -rm -f *.tab.c @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_clock.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_debugging.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_fd.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_flags.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_ignoreset.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_interceptors.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_interceptors_mac.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_interface.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_interface_ann.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_interface_atomic.Plo@am__quote@ @@ -492,7 +503,10 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_aarch64.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_amd64.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_mips64.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_mutex.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_ppc64.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_proc.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_report.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_thread.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_stack_trace.Plo@am__quote@ @@ -548,6 +562,27 @@ mostlyclean-libtool: clean-libtool: -rm -rf .libs _libs +install-nodist_toolexeclibHEADERS: $(nodist_toolexeclib_HEADERS) + @$(NORMAL_INSTALL) + @list='$(nodist_toolexeclib_HEADERS)'; test -n "$(toolexeclibdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(toolexeclibdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(toolexeclibdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(toolexeclibdir)'"; \ + $(INSTALL_HEADER) $$files "$(DESTDIR)$(toolexeclibdir)" || exit $$?; \ + done + +uninstall-nodist_toolexeclibHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(nodist_toolexeclib_HEADERS)'; test -n "$(toolexeclibdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(toolexeclibdir)'; $(am__uninstall_files_from_dir) ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES) list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ @@ -602,9 +637,9 @@ distclean-tags: -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags check-am: all-am check: check-am -all-am: Makefile $(LTLIBRARIES) +all-am: Makefile $(LTLIBRARIES) $(HEADERS) installdirs: - for dir in "$(DESTDIR)$(toolexeclibdir)"; do \ + for dir in "$(DESTDIR)$(toolexeclibdir)" "$(DESTDIR)$(toolexeclibdir)"; do \ test -z "$$dir" || $(MKDIR_P) "$$dir"; \ done install: install-am @@ -666,7 +701,8 @@ install-dvi: install-dvi-am install-dvi-am: -install-exec-am: install-toolexeclibLTLIBRARIES +install-exec-am: install-nodist_toolexeclibHEADERS \ + install-toolexeclibLTLIBRARIES install-html: install-html-am @@ -706,7 +742,8 @@ ps: ps-am ps-am: -uninstall-am: uninstall-toolexeclibLTLIBRARIES +uninstall-am: uninstall-nodist_toolexeclibHEADERS \ + uninstall-toolexeclibLTLIBRARIES .MAKE: install-am install-strip @@ -717,13 +754,18 @@ uninstall-am: uninstall-toolexeclibLTLIBRARIES install-am install-data install-data-am install-dvi \ install-dvi-am install-exec install-exec-am install-html \ install-html-am install-info install-info-am install-man \ - install-pdf install-pdf-am install-ps install-ps-am \ - install-strip install-toolexeclibLTLIBRARIES installcheck \ - installcheck-am installdirs maintainer-clean \ - maintainer-clean-generic mostlyclean mostlyclean-compile \ - mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \ - tags uninstall uninstall-am uninstall-toolexeclibLTLIBRARIES - + install-nodist_toolexeclibHEADERS install-pdf install-pdf-am \ + install-ps install-ps-am install-strip \ + install-toolexeclibLTLIBRARIES installcheck installcheck-am \ + installdirs maintainer-clean maintainer-clean-generic \ + mostlyclean mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool pdf pdf-am ps ps-am tags uninstall \ + uninstall-am uninstall-nodist_toolexeclibHEADERS \ + uninstall-toolexeclibLTLIBRARIES + + +libtsan_preinit.o: tsan_preinit.o + cp $< $@ # Tell versions [3.59,3.63) of GNU make to not export all variables. # Otherwise a system limit (for SysV at least) may be exceeded. diff --git a/libsanitizer/tsan/tsan_clock.cc b/libsanitizer/tsan/tsan_clock.cc index 037afc83fc6c..23f9228a6720 100644 --- a/libsanitizer/tsan/tsan_clock.cc +++ b/libsanitizer/tsan/tsan_clock.cc @@ -80,7 +80,7 @@ // We don't have ThreadState in these methods, so this is an ugly hack that // works only in C++. -#ifndef SANITIZER_GO +#if !SANITIZER_GO # define CPP_STAT_INC(typ) StatInc(cur_thread(), typ) #else # define CPP_STAT_INC(typ) (void)0 diff --git a/libsanitizer/tsan/tsan_debugging.cc b/libsanitizer/tsan/tsan_debugging.cc new file mode 100644 index 000000000000..d26d4828b15d --- /dev/null +++ b/libsanitizer/tsan/tsan_debugging.cc @@ -0,0 +1,160 @@ +//===-- tsan_debugging.cc -------------------------------------------------===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// TSan debugging API implementation. +//===----------------------------------------------------------------------===// +#include "tsan_interface.h" +#include "tsan_report.h" +#include "tsan_rtl.h" + +using namespace __tsan; + +static const char *ReportTypeDescription(ReportType typ) { + if (typ == ReportTypeRace) return "data-race"; + if (typ == ReportTypeVptrRace) return "data-race-vptr"; + if (typ == ReportTypeUseAfterFree) return "heap-use-after-free"; + if (typ == ReportTypeVptrUseAfterFree) return "heap-use-after-free-vptr"; + if (typ == ReportTypeThreadLeak) return "thread-leak"; + if (typ == ReportTypeMutexDestroyLocked) return "locked-mutex-destroy"; + if (typ == ReportTypeMutexDoubleLock) return "mutex-double-lock"; + if (typ == ReportTypeMutexInvalidAccess) return "mutex-invalid-access"; + if (typ == ReportTypeMutexBadUnlock) return "mutex-bad-unlock"; + if (typ == ReportTypeMutexBadReadLock) return "mutex-bad-read-lock"; + if (typ == ReportTypeMutexBadReadUnlock) return "mutex-bad-read-unlock"; + if (typ == ReportTypeSignalUnsafe) return "signal-unsafe-call"; + if (typ == ReportTypeErrnoInSignal) return "errno-in-signal-handler"; + if (typ == ReportTypeDeadlock) return "lock-order-inversion"; + return ""; +} + +static const char *ReportLocationTypeDescription(ReportLocationType typ) { + if (typ == ReportLocationGlobal) return "global"; + if (typ == ReportLocationHeap) return "heap"; + if (typ == ReportLocationStack) return "stack"; + if (typ == ReportLocationTLS) return "tls"; + if (typ == ReportLocationFD) return "fd"; + return ""; +} + +static void CopyTrace(SymbolizedStack *first_frame, void **trace, + uptr trace_size) { + uptr i = 0; + for (SymbolizedStack *frame = first_frame; frame != nullptr; + frame = frame->next) { + trace[i++] = (void *)frame->info.address; + if (i >= trace_size) break; + } +} + +// Meant to be called by the debugger. +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_get_current_report() { + return const_cast<ReportDesc*>(cur_thread()->current_report); +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_data(void *report, const char **description, int *count, + int *stack_count, int *mop_count, int *loc_count, + int *mutex_count, int *thread_count, + int *unique_tid_count, void **sleep_trace, + uptr trace_size) { + const ReportDesc *rep = (ReportDesc *)report; + *description = ReportTypeDescription(rep->typ); + *count = rep->count; + *stack_count = rep->stacks.Size(); + *mop_count = rep->mops.Size(); + *loc_count = rep->locs.Size(); + *mutex_count = rep->mutexes.Size(); + *thread_count = rep->threads.Size(); + *unique_tid_count = rep->unique_tids.Size(); + if (rep->sleep) CopyTrace(rep->sleep->frames, sleep_trace, trace_size); + return 1; +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_stack(void *report, uptr idx, void **trace, + uptr trace_size) { + const ReportDesc *rep = (ReportDesc *)report; + CHECK_LT(idx, rep->stacks.Size()); + ReportStack *stack = rep->stacks[idx]; + if (stack) CopyTrace(stack->frames, trace, trace_size); + return stack ? 1 : 0; +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_mop(void *report, uptr idx, int *tid, void **addr, + int *size, int *write, int *atomic, void **trace, + uptr trace_size) { + const ReportDesc *rep = (ReportDesc *)report; + CHECK_LT(idx, rep->mops.Size()); + ReportMop *mop = rep->mops[idx]; + *tid = mop->tid; + *addr = (void *)mop->addr; + *size = mop->size; + *write = mop->write ? 1 : 0; + *atomic = mop->atomic ? 1 : 0; + if (mop->stack) CopyTrace(mop->stack->frames, trace, trace_size); + return 1; +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_loc(void *report, uptr idx, const char **type, + void **addr, uptr *start, uptr *size, int *tid, + int *fd, int *suppressable, void **trace, + uptr trace_size) { + const ReportDesc *rep = (ReportDesc *)report; + CHECK_LT(idx, rep->locs.Size()); + ReportLocation *loc = rep->locs[idx]; + *type = ReportLocationTypeDescription(loc->type); + *addr = (void *)loc->global.start; + *start = loc->heap_chunk_start; + *size = loc->heap_chunk_size; + *tid = loc->tid; + *fd = loc->fd; + *suppressable = loc->suppressable; + if (loc->stack) CopyTrace(loc->stack->frames, trace, trace_size); + return 1; +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_mutex(void *report, uptr idx, uptr *mutex_id, void **addr, + int *destroyed, void **trace, uptr trace_size) { + const ReportDesc *rep = (ReportDesc *)report; + CHECK_LT(idx, rep->mutexes.Size()); + ReportMutex *mutex = rep->mutexes[idx]; + *mutex_id = mutex->id; + *addr = (void *)mutex->addr; + *destroyed = mutex->destroyed; + if (mutex->stack) CopyTrace(mutex->stack->frames, trace, trace_size); + return 1; +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_thread(void *report, uptr idx, int *tid, uptr *os_id, + int *running, const char **name, int *parent_tid, + void **trace, uptr trace_size) { + const ReportDesc *rep = (ReportDesc *)report; + CHECK_LT(idx, rep->threads.Size()); + ReportThread *thread = rep->threads[idx]; + *tid = thread->id; + *os_id = thread->os_id; + *running = thread->running; + *name = thread->name; + *parent_tid = thread->parent_tid; + if (thread->stack) CopyTrace(thread->stack->frames, trace, trace_size); + return 1; +} + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_unique_tid(void *report, uptr idx, int *tid) { + const ReportDesc *rep = (ReportDesc *)report; + CHECK_LT(idx, rep->unique_tids.Size()); + *tid = rep->unique_tids[idx]; + return 1; +} diff --git a/libsanitizer/tsan/tsan_defs.h b/libsanitizer/tsan/tsan_defs.h index 259b30bee5db..e540526caa22 100644 --- a/libsanitizer/tsan/tsan_defs.h +++ b/libsanitizer/tsan/tsan_defs.h @@ -27,27 +27,18 @@ #endif #ifndef TSAN_CONTAINS_UBSAN -# define TSAN_CONTAINS_UBSAN (CAN_SANITIZE_UB && !defined(SANITIZER_GO)) +# if CAN_SANITIZE_UB && !SANITIZER_GO +# define TSAN_CONTAINS_UBSAN 1 +# else +# define TSAN_CONTAINS_UBSAN 0 +# endif #endif namespace __tsan { -#ifdef SANITIZER_GO -const bool kGoMode = true; -const bool kCppMode = false; -const char *const kTsanOptionsEnv = "GORACE"; -// Go linker does not support weak symbols. -#define CPP_WEAK -#else -const bool kGoMode = false; -const bool kCppMode = true; -const char *const kTsanOptionsEnv = "TSAN_OPTIONS"; -#define CPP_WEAK WEAK -#endif - const int kTidBits = 13; const unsigned kMaxTid = 1 << kTidBits; -#ifndef SANITIZER_GO +#if !SANITIZER_GO const unsigned kMaxTidInClock = kMaxTid * 2; // This includes msb 'freed' bit. #else const unsigned kMaxTidInClock = kMaxTid; // Go does not track freed memory. @@ -146,6 +137,7 @@ struct MD5Hash { MD5Hash md5_hash(const void *data, uptr size); +struct Processor; struct ThreadState; class ThreadContext; struct Context; diff --git a/libsanitizer/tsan/tsan_dense_alloc.h b/libsanitizer/tsan/tsan_dense_alloc.h index 651c112c78ae..780ff6f4c8e5 100644 --- a/libsanitizer/tsan/tsan_dense_alloc.h +++ b/libsanitizer/tsan/tsan_dense_alloc.h @@ -106,7 +106,7 @@ class DenseSlabAlloc { // Reserve 0 as invalid index. IndexT start = fillpos_ == 0 ? 1 : 0; for (IndexT i = start; i < kL2Size; i++) { - new(batch + i) T(); + new(batch + i) T; *(IndexT*)(batch + i) = i + 1 + fillpos_ * kL2Size; } *(IndexT*)(batch + kL2Size - 1) = 0; diff --git a/libsanitizer/tsan/tsan_flags.cc b/libsanitizer/tsan/tsan_flags.cc index 4cec0ac09f41..5064a0fe6539 100644 --- a/libsanitizer/tsan/tsan_flags.cc +++ b/libsanitizer/tsan/tsan_flags.cc @@ -27,8 +27,8 @@ Flags *flags() { #ifdef TSAN_EXTERNAL_HOOKS extern "C" const char* __tsan_default_options(); #else -extern "C" SANITIZER_INTERFACE_ATTRIBUTE -const char *WEAK __tsan_default_options() { +SANITIZER_WEAK_DEFAULT_IMPL +const char *__tsan_default_options() { return ""; } #endif @@ -59,7 +59,7 @@ void InitializeFlags(Flags *f, const char *env) { CommonFlags cf; cf.CopyFrom(*common_flags()); cf.allow_addr2line = true; - if (kGoMode) { + if (SANITIZER_GO) { // Does not work as expected for Go: runtime handles SIGABRT and crashes. cf.abort_on_error = false; // Go does not have mutexes. @@ -69,6 +69,7 @@ void InitializeFlags(Flags *f, const char *env) { cf.print_suppressions = false; cf.stack_trace_format = " #%n %f %S %M"; cf.exitcode = 66; + cf.intercept_tls_get_addr = true; OverrideCommonFlags(cf); } @@ -106,7 +107,7 @@ void InitializeFlags(Flags *f, const char *env) { f->report_signal_unsafe = false; } - SetVerbosity(common_flags()->verbosity); + InitializeCommonFlags(); if (Verbosity()) ReportUnrecognizedFlags(); diff --git a/libsanitizer/tsan/tsan_flags.inc b/libsanitizer/tsan/tsan_flags.inc index 822e560b6223..78f5e80fd5cc 100644 --- a/libsanitizer/tsan/tsan_flags.inc +++ b/libsanitizer/tsan/tsan_flags.inc @@ -59,8 +59,9 @@ TSAN_FLAG(bool, stop_on_start, false, "Stops on start until __tsan_resume() is called (for debugging).") TSAN_FLAG(bool, running_on_valgrind, false, "Controls whether RunningOnValgrind() returns true or false.") +// There are a lot of goroutines in Go, so we use smaller history. TSAN_FLAG( - int, history_size, kGoMode ? 1 : 3, // There are a lot of goroutines in Go. + int, history_size, SANITIZER_GO ? 1 : 3, "Per-thread history size, controls how many previous memory accesses " "are remembered per thread. Possible values are [0..7]. " "history_size=0 amounts to 32K memory accesses. Each next value doubles " @@ -74,3 +75,7 @@ TSAN_FLAG(int, io_sync, 1, TSAN_FLAG(bool, die_after_fork, true, "Die after multi-threaded fork if the child creates new threads.") TSAN_FLAG(const char *, suppressions, "", "Suppressions file name.") +TSAN_FLAG(bool, ignore_interceptors_accesses, false, + "Ignore reads and writes from all interceptors.") +TSAN_FLAG(bool, shared_ptr_interceptor, true, + "Track atomic reference counting in libc++ shared_ptr and weak_ptr.") diff --git a/libsanitizer/tsan/tsan_interceptors.cc b/libsanitizer/tsan/tsan_interceptors.cc index 0c71c008d704..bf5f2d5b66ce 100644 --- a/libsanitizer/tsan/tsan_interceptors.cc +++ b/libsanitizer/tsan/tsan_interceptors.cc @@ -17,6 +17,7 @@ #include "sanitizer_common/sanitizer_platform_limits_posix.h" #include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_stacktrace.h" +#include "sanitizer_common/sanitizer_tls_get_addr.h" #include "interception/interception.h" #include "tsan_interceptors.h" #include "tsan_interface.h" @@ -38,14 +39,9 @@ using namespace __tsan; // NOLINT #define stderr __stderrp #endif -#if SANITIZER_FREEBSD -#define __libc_realloc __realloc -#define __libc_calloc __calloc -#elif SANITIZER_MAC -#define __libc_malloc REAL(malloc) -#define __libc_realloc REAL(realloc) -#define __libc_calloc REAL(calloc) -#define __libc_free REAL(free) +#if SANITIZER_ANDROID +#define __errno_location __errno +#define mallopt(a, b) #endif #if SANITIZER_LINUX || SANITIZER_FREEBSD @@ -77,9 +73,9 @@ struct ucontext_t { }; #endif -#if defined(__x86_64__) || defined(__mips__) +#if defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1 #define PTHREAD_ABI_BASE "GLIBC_2.3.2" -#elif defined(__aarch64__) +#elif defined(__aarch64__) || SANITIZER_PPC64V2 #define PTHREAD_ABI_BASE "GLIBC_2.17" #endif @@ -90,10 +86,6 @@ extern "C" int pthread_attr_setstacksize(void *attr, uptr stacksize); extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v)); extern "C" int pthread_setspecific(unsigned key, const void *v); DECLARE_REAL(int, pthread_mutexattr_gettype, void *, void *) -extern "C" int pthread_sigmask(int how, const __sanitizer_sigset_t *set, - __sanitizer_sigset_t *oldset); -// REAL(sigfillset) defined in common interceptors. -DECLARE_REAL(int, sigfillset, __sanitizer_sigset_t *set) DECLARE_REAL(int, fflush, __sanitizer_FILE *fp) DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size) DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) @@ -101,19 +93,22 @@ extern "C" void *pthread_self(); extern "C" void _exit(int status); extern "C" int *__errno_location(); extern "C" int fileno_unlocked(void *stream); -extern "C" void *__libc_calloc(uptr size, uptr n); -extern "C" void *__libc_realloc(void *ptr, uptr size); extern "C" int dirfd(void *dirp); -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID extern "C" int mallopt(int param, int value); #endif extern __sanitizer_FILE *stdout, *stderr; +#if !SANITIZER_FREEBSD && !SANITIZER_MAC const int PTHREAD_MUTEX_RECURSIVE = 1; const int PTHREAD_MUTEX_RECURSIVE_NP = 1; +#else +const int PTHREAD_MUTEX_RECURSIVE = 2; +const int PTHREAD_MUTEX_RECURSIVE_NP = 2; +#endif const int EINVAL = 22; const int EBUSY = 16; const int EOWNERDEAD = 130; -#if !SANITIZER_MAC +#if !SANITIZER_FREEBSD && !SANITIZER_MAC const int EPOLL_CTL_ADD = 1; #endif const int SIGILL = 4; @@ -122,7 +117,7 @@ const int SIGFPE = 8; const int SIGSEGV = 11; const int SIGPIPE = 13; const int SIGTERM = 15; -#ifdef __mips__ +#if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_MAC const int SIGBUS = 10; const int SIGSYS = 12; #else @@ -147,6 +142,17 @@ typedef long long_t; // NOLINT typedef void (*sighandler_t)(int sig); typedef void (*sigactionhandler_t)(int sig, my_siginfo_t *siginfo, void *uctx); +#if SANITIZER_ANDROID +struct sigaction_t { + u32 sa_flags; + union { + sighandler_t sa_handler; + sigactionhandler_t sa_sigaction; + }; + __sanitizer_sigset_t sa_mask; + void (*sa_restorer)(); +}; +#else struct sigaction_t { #ifdef __mips__ u32 sa_flags; @@ -158,6 +164,9 @@ struct sigaction_t { #if SANITIZER_FREEBSD int sa_flags; __sanitizer_sigset_t sa_mask; +#elif SANITIZER_MAC + __sanitizer_sigset_t sa_mask; + int sa_flags; #else __sanitizer_sigset_t sa_mask; #ifndef __mips__ @@ -166,11 +175,12 @@ struct sigaction_t { void (*sa_restorer)(); #endif }; +#endif const sighandler_t SIG_DFL = (sighandler_t)0; const sighandler_t SIG_IGN = (sighandler_t)1; const sighandler_t SIG_ERR = (sighandler_t)-1; -#if SANITIZER_FREEBSD +#if SANITIZER_FREEBSD || SANITIZER_MAC const int SA_SIGINFO = 0x40; const int SIG_SETMASK = 3; #elif defined(__mips__) @@ -199,6 +209,9 @@ struct ThreadSignalContext { atomic_uintptr_t in_blocking_func; atomic_uintptr_t have_pending_signals; SignalDesc pending_signals[kSigCount]; + // emptyset and oldset are too big for stack. + __sanitizer_sigset_t emptyset; + __sanitizer_sigset_t oldset; }; // The object is 64-byte aligned, because we want hot data to be located in @@ -231,26 +244,33 @@ static ThreadSignalContext *SigCtx(ThreadState *thr) { return ctx; } +#if !SANITIZER_MAC static unsigned g_thread_finalize_key; +#endif ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc) : thr_(thr) , pc_(pc) , in_ignored_lib_(false) { - if (!thr_->ignore_interceptors) { - Initialize(thr); + Initialize(thr); + if (!thr_->is_inited) + return; + if (!thr_->ignore_interceptors) FuncEntry(thr, pc); - } DPrintf("#%d: intercept %s()\n", thr_->tid, fname); if (!thr_->in_ignored_lib && libignore()->IsIgnored(pc)) { in_ignored_lib_ = true; thr_->in_ignored_lib = true; ThreadIgnoreBegin(thr_, pc_); } + if (flags()->ignore_interceptors_accesses) ThreadIgnoreBegin(thr_, pc_); } ScopedInterceptor::~ScopedInterceptor() { + if (!thr_->is_inited) + return; + if (flags()->ignore_interceptors_accesses) ThreadIgnoreEnd(thr_, pc_); if (in_ignored_lib_) { thr_->in_ignored_lib = false; ThreadIgnoreEnd(thr_, pc_); @@ -262,6 +282,22 @@ ScopedInterceptor::~ScopedInterceptor() { } } +void ScopedInterceptor::UserCallbackStart() { + if (flags()->ignore_interceptors_accesses) ThreadIgnoreEnd(thr_, pc_); + if (in_ignored_lib_) { + thr_->in_ignored_lib = false; + ThreadIgnoreEnd(thr_, pc_); + } +} + +void ScopedInterceptor::UserCallbackEnd() { + if (in_ignored_lib_) { + thr_->in_ignored_lib = true; + ThreadIgnoreBegin(thr_, pc_); + } + if (flags()->ignore_interceptors_accesses) ThreadIgnoreBegin(thr_, pc_); +} + #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func) #if SANITIZER_FREEBSD # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func) @@ -340,12 +376,13 @@ static void at_exit_wrapper(void *arg) { Acquire(thr, pc, (uptr)arg); AtExitCtx *ctx = (AtExitCtx*)arg; ((void(*)(void *arg))ctx->f)(ctx->arg); - __libc_free(ctx); + InternalFree(ctx); } static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), void *arg, void *dso); +#if !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, atexit, void (*f)()) { if (cur_thread()->in_symbolizer) return 0; @@ -354,6 +391,7 @@ TSAN_INTERCEPTOR(int, atexit, void (*f)()) { SCOPED_INTERCEPTOR_RAW(atexit, f); return setup_at_exit_wrapper(thr, pc, (void(*)())f, 0, 0); } +#endif TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) { if (cur_thread()->in_symbolizer) @@ -364,7 +402,7 @@ TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) { static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), void *arg, void *dso) { - AtExitCtx *ctx = (AtExitCtx*)__libc_malloc(sizeof(AtExitCtx)); + AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx)); ctx->f = f; ctx->arg = arg; Release(thr, pc, (uptr)ctx); @@ -383,14 +421,14 @@ static void on_exit_wrapper(int status, void *arg) { Acquire(thr, pc, (uptr)arg); AtExitCtx *ctx = (AtExitCtx*)arg; ((void(*)(int status, void *arg))ctx->f)(status, ctx->arg); - __libc_free(ctx); + InternalFree(ctx); } TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) { if (cur_thread()->in_symbolizer) return 0; SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg); - AtExitCtx *ctx = (AtExitCtx*)__libc_malloc(sizeof(AtExitCtx)); + AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx)); ctx->f = (void(*)())f; ctx->arg = arg; Release(thr, pc, (uptr)ctx); @@ -409,7 +447,7 @@ static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) { JmpBuf *buf = &thr->jmp_bufs[i]; if (buf->sp <= sp) { uptr sz = thr->jmp_bufs.Size(); - thr->jmp_bufs[i] = thr->jmp_bufs[sz - 1]; + internal_memcpy(buf, &thr->jmp_bufs[sz - 1], sizeof(*buf)); thr->jmp_bufs.PopBack(); i--; } @@ -436,15 +474,19 @@ static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) { } static void LongJmp(ThreadState *thr, uptr *env) { -#if SANITIZER_FREEBSD +#ifdef __powerpc__ + uptr mangled_sp = env[0]; +#elif SANITIZER_FREEBSD || SANITIZER_MAC uptr mangled_sp = env[2]; #elif defined(SANITIZER_LINUX) # ifdef __aarch64__ uptr mangled_sp = env[13]; +# elif defined(__mips64) + uptr mangled_sp = env[1]; # else uptr mangled_sp = env[6]; # endif -#endif // SANITIZER_FREEBSD +#endif // Find the saved buf by mangled_sp. for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) { JmpBuf *buf = &thr->jmp_bufs[i]; @@ -474,6 +516,11 @@ extern "C" void __tsan_setjmp(uptr sp, uptr mangled_sp) { SetJmp(cur_thread(), sp, mangled_sp); } +#if SANITIZER_MAC +TSAN_INTERCEPTOR(int, setjmp, void *env); +TSAN_INTERCEPTOR(int, _setjmp, void *env); +TSAN_INTERCEPTOR(int, sigsetjmp, void *env); +#else // SANITIZER_MAC // Not called. Merely to satisfy TSAN_INTERCEPT(). extern "C" SANITIZER_INTERFACE_ATTRIBUTE int __interceptor_setjmp(void *env); @@ -512,10 +559,14 @@ DEFINE_REAL(int, setjmp, void *env) DEFINE_REAL(int, _setjmp, void *env) DEFINE_REAL(int, sigsetjmp, void *env) DEFINE_REAL(int, __sigsetjmp, void *env) +#endif // SANITIZER_MAC TSAN_INTERCEPTOR(void, longjmp, uptr *env, int val) { + // Note: if we call REAL(longjmp) in the context of ScopedInterceptor, + // bad things will happen. We will jump over ScopedInterceptor dtor and can + // leave thr->in_ignored_lib set. { - SCOPED_TSAN_INTERCEPTOR(longjmp, env, val); + SCOPED_INTERCEPTOR_RAW(longjmp, env, val); } LongJmp(cur_thread(), env); REAL(longjmp)(env, val); @@ -523,7 +574,7 @@ TSAN_INTERCEPTOR(void, longjmp, uptr *env, int val) { TSAN_INTERCEPTOR(void, siglongjmp, uptr *env, int val) { { - SCOPED_TSAN_INTERCEPTOR(siglongjmp, env, val); + SCOPED_INTERCEPTOR_RAW(siglongjmp, env, val); } LongJmp(cur_thread(), env); REAL(siglongjmp)(env, val); @@ -532,7 +583,7 @@ TSAN_INTERCEPTOR(void, siglongjmp, uptr *env, int val) { #if !SANITIZER_MAC TSAN_INTERCEPTOR(void*, malloc, uptr size) { if (cur_thread()->in_symbolizer) - return __libc_malloc(size); + return InternalAlloc(size); void *p = 0; { SCOPED_INTERCEPTOR_RAW(malloc, size); @@ -549,7 +600,7 @@ TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) { TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { if (cur_thread()->in_symbolizer) - return __libc_calloc(size, n); + return InternalCalloc(size, n); void *p = 0; { SCOPED_INTERCEPTOR_RAW(calloc, size, n); @@ -561,7 +612,7 @@ TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { if (cur_thread()->in_symbolizer) - return __libc_realloc(p, size); + return InternalRealloc(p, size); if (p) invoke_free_hook(p); { @@ -576,7 +627,7 @@ TSAN_INTERCEPTOR(void, free, void *p) { if (p == 0) return; if (cur_thread()->in_symbolizer) - return __libc_free(p); + return InternalFree(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(free, p); user_free(thr, pc, p); @@ -586,7 +637,7 @@ TSAN_INTERCEPTOR(void, cfree, void *p) { if (p == 0) return; if (cur_thread()->in_symbolizer) - return __libc_free(p); + return InternalFree(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(cfree, p); user_free(thr, pc, p); @@ -598,69 +649,6 @@ TSAN_INTERCEPTOR(uptr, malloc_usable_size, void *p) { } #endif -TSAN_INTERCEPTOR(uptr, strlen, const char *s) { - SCOPED_TSAN_INTERCEPTOR(strlen, s); - uptr len = internal_strlen(s); - MemoryAccessRange(thr, pc, (uptr)s, len + 1, false); - return len; -} - -TSAN_INTERCEPTOR(void*, memset, void *dst, int v, uptr size) { - // On FreeBSD we get here from libthr internals on thread initialization. - if (!COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) { - SCOPED_TSAN_INTERCEPTOR(memset, dst, v, size); - MemoryAccessRange(thr, pc, (uptr)dst, size, true); - } - return internal_memset(dst, v, size); -} - -TSAN_INTERCEPTOR(void*, memcpy, void *dst, const void *src, uptr size) { - // On FreeBSD we get here from libthr internals on thread initialization. - if (!COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) { - SCOPED_TSAN_INTERCEPTOR(memcpy, dst, src, size); - MemoryAccessRange(thr, pc, (uptr)dst, size, true); - MemoryAccessRange(thr, pc, (uptr)src, size, false); - } - // On OS X, calling internal_memcpy here will cause memory corruptions, - // because memcpy and memmove are actually aliases of the same implementation. - // We need to use internal_memmove here. - return internal_memmove(dst, src, size); -} - -TSAN_INTERCEPTOR(void*, memmove, void *dst, void *src, uptr n) { - if (!COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) { - SCOPED_TSAN_INTERCEPTOR(memmove, dst, src, n); - MemoryAccessRange(thr, pc, (uptr)dst, n, true); - MemoryAccessRange(thr, pc, (uptr)src, n, false); - } - return REAL(memmove)(dst, src, n); -} - -TSAN_INTERCEPTOR(char*, strchr, char *s, int c) { - SCOPED_TSAN_INTERCEPTOR(strchr, s, c); - char *res = REAL(strchr)(s, c); - uptr len = internal_strlen(s); - uptr n = res ? (char*)res - (char*)s + 1 : len + 1; - READ_STRING_OF_LEN(thr, pc, s, len, n); - return res; -} - -#if !SANITIZER_MAC -TSAN_INTERCEPTOR(char*, strchrnul, char *s, int c) { - SCOPED_TSAN_INTERCEPTOR(strchrnul, s, c); - char *res = REAL(strchrnul)(s, c); - uptr len = (char*)res - (char*)s + 1; - READ_STRING(thr, pc, s, len); - return res; -} -#endif - -TSAN_INTERCEPTOR(char*, strrchr, char *s, int c) { - SCOPED_TSAN_INTERCEPTOR(strrchr, s, c); - MemoryAccessRange(thr, pc, (uptr)s, internal_strlen(s) + 1, false); - return REAL(strrchr)(s, c); -} - TSAN_INTERCEPTOR(char*, strcpy, char *dst, const char *src) { // NOLINT SCOPED_TSAN_INTERCEPTOR(strcpy, dst, src); // NOLINT uptr srclen = internal_strlen(src); @@ -706,7 +694,11 @@ TSAN_INTERCEPTOR(void *, mmap, void *addr, SIZE_T sz, int prot, int flags, if (res != MAP_FAILED) { if (fd > 0) FdAccess(thr, pc, fd); - MemoryRangeImitateWrite(thr, pc, (uptr)res, sz); + + if (thr->ignore_reads_and_writes == 0) + MemoryRangeImitateWrite(thr, pc, (uptr)res, sz); + else + MemoryResetRange(thr, pc, (uptr)res, sz); } return res; } @@ -721,7 +713,11 @@ TSAN_INTERCEPTOR(void *, mmap64, void *addr, SIZE_T sz, int prot, int flags, if (res != MAP_FAILED) { if (fd > 0) FdAccess(thr, pc, fd); - MemoryRangeImitateWrite(thr, pc, (uptr)res, sz); + + if (thr->ignore_reads_and_writes == 0) + MemoryRangeImitateWrite(thr, pc, (uptr)res, sz); + else + MemoryResetRange(thr, pc, (uptr)res, sz); } return res; } @@ -735,7 +731,8 @@ TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) { if (sz != 0) { // If sz == 0, munmap will return EINVAL and don't unmap any memory. DontNeedShadowFor((uptr)addr, sz); - ctx->metamap.ResetRange(thr, pc, (uptr)addr, (uptr)sz); + ScopedGlobalProcessor sgp; + ctx->metamap.ResetRange(thr->proc(), (uptr)addr, (uptr)sz); } int res = REAL(munmap)(addr, sz); return res; @@ -782,8 +779,25 @@ TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) { } #endif +// __cxa_guard_acquire and friends need to be intercepted in a special way - +// regular interceptors will break statically-linked libstdc++. Linux +// interceptors are especially defined as weak functions (so that they don't +// cause link errors when user defines them as well). So they silently +// auto-disable themselves when such symbol is already present in the binary. If +// we link libstdc++ statically, it will bring own __cxa_guard_acquire which +// will silently replace our interceptor. That's why on Linux we simply export +// these interceptors with INTERFACE_ATTRIBUTE. +// On OS X, we don't support statically linking, so we just use a regular +// interceptor. +#if SANITIZER_MAC +#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR +#else +#define STDCXX_INTERCEPTOR(rettype, name, ...) \ + extern "C" rettype INTERFACE_ATTRIBUTE name(__VA_ARGS__) +#endif + // Used in thread-safe function static initialization. -extern "C" int INTERFACE_ATTRIBUTE __cxa_guard_acquire(atomic_uint32_t *g) { +STDCXX_INTERCEPTOR(int, __cxa_guard_acquire, atomic_uint32_t *g) { SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g); for (;;) { u32 cmp = atomic_load(g, memory_order_acquire); @@ -799,13 +813,13 @@ extern "C" int INTERFACE_ATTRIBUTE __cxa_guard_acquire(atomic_uint32_t *g) { } } -extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_release(atomic_uint32_t *g) { +STDCXX_INTERCEPTOR(void, __cxa_guard_release, atomic_uint32_t *g) { SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g); Release(thr, pc, (uptr)g); atomic_store(g, 1, memory_order_release); } -extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_abort(atomic_uint32_t *g) { +STDCXX_INTERCEPTOR(void, __cxa_guard_abort, atomic_uint32_t *g) { SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g); atomic_store(g, 0, memory_order_relaxed); } @@ -813,16 +827,21 @@ extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_abort(atomic_uint32_t *g) { namespace __tsan { void DestroyThreadState() { ThreadState *thr = cur_thread(); + Processor *proc = thr->proc(); ThreadFinish(thr); + ProcUnwire(proc, thr); + ProcDestroy(proc); ThreadSignalContext *sctx = thr->signal_ctx; if (sctx) { thr->signal_ctx = 0; UnmapOrDie(sctx, sizeof(*sctx)); } + DTLS_Destroy(); cur_thread_finalize(); } } // namespace __tsan +#if !SANITIZER_MAC static void thread_finalize(void *v) { uptr iter = (uptr)v; if (iter > 1) { @@ -834,6 +853,7 @@ static void thread_finalize(void *v) { } DestroyThreadState(); } +#endif struct ThreadParam { @@ -851,6 +871,7 @@ extern "C" void *__tsan_thread_start_func(void *arg) { ThreadState *thr = cur_thread(); // Thread-local state is not initialized yet. ScopedIgnoreInterceptors ignore; +#if !SANITIZER_MAC ThreadIgnoreBegin(thr, 0); if (pthread_setspecific(g_thread_finalize_key, (void *)GetPthreadDestructorIterations())) { @@ -858,8 +879,11 @@ extern "C" void *__tsan_thread_start_func(void *arg) { Die(); } ThreadIgnoreEnd(thr, 0); +#endif while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0) internal_sched_yield(); + Processor *proc = ProcCreate(); + ProcWire(proc, thr); ThreadStart(thr, tid, GetTid()); atomic_store(&p->tid, 0, memory_order_release); } @@ -1017,12 +1041,12 @@ INTERCEPTOR(int, pthread_cond_init, void *c, void *a) { return REAL(pthread_cond_init)(cond, a); } -INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) { - void *cond = init_cond(c); - SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait, cond, m); +static int cond_wait(ThreadState *thr, uptr pc, ScopedInterceptor *si, + int (*fn)(void *c, void *m, void *abstime), void *c, + void *m, void *t) { MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); MutexUnlock(thr, pc, (uptr)m); - CondMutexUnlockCtx arg = {&si, thr, pc, m}; + CondMutexUnlockCtx arg = {si, thr, pc, m}; int res = 0; // This ensures that we handle mutex lock even in case of pthread_cancel. // See test/tsan/cond_cancel.cc. @@ -1030,36 +1054,38 @@ INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) { // Enable signal delivery while the thread is blocked. BlockingCall bc(thr); res = call_pthread_cancel_with_cleanup( - (int(*)(void *c, void *m, void *abstime))REAL(pthread_cond_wait), - cond, m, 0, (void(*)(void *arg))cond_mutex_unlock, &arg); + fn, c, m, t, (void (*)(void *arg))cond_mutex_unlock, &arg); } - if (res == errno_EOWNERDEAD) - MutexRepair(thr, pc, (uptr)m); + if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m); MutexLock(thr, pc, (uptr)m); return res; } +INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) { + void *cond = init_cond(c); + SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait, cond, m); + return cond_wait(thr, pc, &si, (int (*)(void *c, void *m, void *abstime))REAL( + pthread_cond_wait), + cond, m, 0); +} + INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, void *abstime) { void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, cond, m, abstime); - MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); - MutexUnlock(thr, pc, (uptr)m); - CondMutexUnlockCtx arg = {&si, thr, pc, m}; - int res = 0; - // This ensures that we handle mutex lock even in case of pthread_cancel. - // See test/tsan/cond_cancel.cc. - { - BlockingCall bc(thr); - res = call_pthread_cancel_with_cleanup( - REAL(pthread_cond_timedwait), cond, m, abstime, - (void(*)(void *arg))cond_mutex_unlock, &arg); - } - if (res == errno_EOWNERDEAD) - MutexRepair(thr, pc, (uptr)m); - MutexLock(thr, pc, (uptr)m); - return res; + return cond_wait(thr, pc, &si, REAL(pthread_cond_timedwait), cond, m, + abstime); } +#if SANITIZER_MAC +INTERCEPTOR(int, pthread_cond_timedwait_relative_np, void *c, void *m, + void *reltime) { + void *cond = init_cond(c); + SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait_relative_np, cond, m, reltime); + return cond_wait(thr, pc, &si, REAL(pthread_cond_timedwait_relative_np), cond, + m, reltime); +} +#endif + INTERCEPTOR(int, pthread_cond_signal, void *c) { void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal, cond); @@ -1316,97 +1342,7 @@ TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) { return 0; } -#if SANITIZER_LINUX -TSAN_INTERCEPTOR(int, __xstat, int version, const char *path, void *buf) { - SCOPED_TSAN_INTERCEPTOR(__xstat, version, path, buf); - READ_STRING(thr, pc, path, 0); - return REAL(__xstat)(version, path, buf); -} -#define TSAN_MAYBE_INTERCEPT___XSTAT TSAN_INTERCEPT(__xstat) -#else -#define TSAN_MAYBE_INTERCEPT___XSTAT -#endif - -TSAN_INTERCEPTOR(int, stat, const char *path, void *buf) { -#if SANITIZER_FREEBSD || SANITIZER_MAC - SCOPED_TSAN_INTERCEPTOR(stat, path, buf); - READ_STRING(thr, pc, path, 0); - return REAL(stat)(path, buf); -#else - SCOPED_TSAN_INTERCEPTOR(__xstat, 0, path, buf); - READ_STRING(thr, pc, path, 0); - return REAL(__xstat)(0, path, buf); -#endif -} - -#if SANITIZER_LINUX -TSAN_INTERCEPTOR(int, __xstat64, int version, const char *path, void *buf) { - SCOPED_TSAN_INTERCEPTOR(__xstat64, version, path, buf); - READ_STRING(thr, pc, path, 0); - return REAL(__xstat64)(version, path, buf); -} -#define TSAN_MAYBE_INTERCEPT___XSTAT64 TSAN_INTERCEPT(__xstat64) -#else -#define TSAN_MAYBE_INTERCEPT___XSTAT64 -#endif - -#if SANITIZER_LINUX -TSAN_INTERCEPTOR(int, stat64, const char *path, void *buf) { - SCOPED_TSAN_INTERCEPTOR(__xstat64, 0, path, buf); - READ_STRING(thr, pc, path, 0); - return REAL(__xstat64)(0, path, buf); -} -#define TSAN_MAYBE_INTERCEPT_STAT64 TSAN_INTERCEPT(stat64) -#else -#define TSAN_MAYBE_INTERCEPT_STAT64 -#endif - -#if SANITIZER_LINUX -TSAN_INTERCEPTOR(int, __lxstat, int version, const char *path, void *buf) { - SCOPED_TSAN_INTERCEPTOR(__lxstat, version, path, buf); - READ_STRING(thr, pc, path, 0); - return REAL(__lxstat)(version, path, buf); -} -#define TSAN_MAYBE_INTERCEPT___LXSTAT TSAN_INTERCEPT(__lxstat) -#else -#define TSAN_MAYBE_INTERCEPT___LXSTAT -#endif - -TSAN_INTERCEPTOR(int, lstat, const char *path, void *buf) { -#if SANITIZER_FREEBSD || SANITIZER_MAC - SCOPED_TSAN_INTERCEPTOR(lstat, path, buf); - READ_STRING(thr, pc, path, 0); - return REAL(lstat)(path, buf); -#else - SCOPED_TSAN_INTERCEPTOR(__lxstat, 0, path, buf); - READ_STRING(thr, pc, path, 0); - return REAL(__lxstat)(0, path, buf); -#endif -} - -#if SANITIZER_LINUX -TSAN_INTERCEPTOR(int, __lxstat64, int version, const char *path, void *buf) { - SCOPED_TSAN_INTERCEPTOR(__lxstat64, version, path, buf); - READ_STRING(thr, pc, path, 0); - return REAL(__lxstat64)(version, path, buf); -} -#define TSAN_MAYBE_INTERCEPT___LXSTAT64 TSAN_INTERCEPT(__lxstat64) -#else -#define TSAN_MAYBE_INTERCEPT___LXSTAT64 -#endif - -#if SANITIZER_LINUX -TSAN_INTERCEPTOR(int, lstat64, const char *path, void *buf) { - SCOPED_TSAN_INTERCEPTOR(__lxstat64, 0, path, buf); - READ_STRING(thr, pc, path, 0); - return REAL(__lxstat64)(0, path, buf); -} -#define TSAN_MAYBE_INTERCEPT_LSTAT64 TSAN_INTERCEPT(lstat64) -#else -#define TSAN_MAYBE_INTERCEPT_LSTAT64 -#endif - -#if SANITIZER_LINUX +#if SANITIZER_LINUX && !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) { SCOPED_TSAN_INTERCEPTOR(__fxstat, version, fd, buf); if (fd > 0) @@ -1419,7 +1355,7 @@ TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) { #endif TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) { -#if SANITIZER_FREEBSD || SANITIZER_MAC +#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf); if (fd > 0) FdAccess(thr, pc, fd); @@ -1432,7 +1368,7 @@ TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) { #endif } -#if SANITIZER_LINUX +#if SANITIZER_LINUX && !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) { SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf); if (fd > 0) @@ -1444,7 +1380,7 @@ TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) { #define TSAN_MAYBE_INTERCEPT___FXSTAT64 #endif -#if SANITIZER_LINUX +#if SANITIZER_LINUX && !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) { SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf); if (fd > 0) @@ -1623,32 +1559,6 @@ TSAN_INTERCEPTOR(int, listen, int fd, int backlog) { return res; } -#if SANITIZER_LINUX -TSAN_INTERCEPTOR(int, epoll_create, int size) { - SCOPED_TSAN_INTERCEPTOR(epoll_create, size); - int fd = REAL(epoll_create)(size); - if (fd >= 0) - FdPollCreate(thr, pc, fd); - return fd; -} -#define TSAN_MAYBE_INTERCEPT_EPOLL_CREATE TSAN_INTERCEPT(epoll_create) -#else -#define TSAN_MAYBE_INTERCEPT_EPOLL_CREATE -#endif - -#if SANITIZER_LINUX -TSAN_INTERCEPTOR(int, epoll_create1, int flags) { - SCOPED_TSAN_INTERCEPTOR(epoll_create1, flags); - int fd = REAL(epoll_create1)(flags); - if (fd >= 0) - FdPollCreate(thr, pc, fd); - return fd; -} -#define TSAN_MAYBE_INTERCEPT_EPOLL_CREATE1 TSAN_INTERCEPT(epoll_create1) -#else -#define TSAN_MAYBE_INTERCEPT_EPOLL_CREATE1 -#endif - TSAN_INTERCEPTOR(int, close, int fd) { SCOPED_TSAN_INTERCEPTOR(close, fd); if (fd >= 0) @@ -1669,7 +1579,7 @@ TSAN_INTERCEPTOR(int, __close, int fd) { #endif // glibc guts -#if SANITIZER_LINUX +#if SANITIZER_LINUX && !SANITIZER_ANDROID TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) { SCOPED_TSAN_INTERCEPTOR(__res_iclose, state, free_addr); int fds[64]; @@ -1703,37 +1613,6 @@ TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) { } #endif -TSAN_INTERCEPTOR(long_t, send, int fd, void *buf, long_t len, int flags) { - SCOPED_TSAN_INTERCEPTOR(send, fd, buf, len, flags); - if (fd >= 0) { - FdAccess(thr, pc, fd); - FdRelease(thr, pc, fd); - } - int res = REAL(send)(fd, buf, len, flags); - return res; -} - -TSAN_INTERCEPTOR(long_t, sendmsg, int fd, void *msg, int flags) { - SCOPED_TSAN_INTERCEPTOR(sendmsg, fd, msg, flags); - if (fd >= 0) { - FdAccess(thr, pc, fd); - FdRelease(thr, pc, fd); - } - int res = REAL(sendmsg)(fd, msg, flags); - return res; -} - -TSAN_INTERCEPTOR(long_t, recv, int fd, void *buf, long_t len, int flags) { - SCOPED_TSAN_INTERCEPTOR(recv, fd, buf, len, flags); - if (fd >= 0) - FdAccess(thr, pc, fd); - int res = REAL(recv)(fd, buf, len, flags); - if (res >= 0 && fd >= 0) { - FdAcquire(thr, pc, fd); - } - return res; -} - TSAN_INTERCEPTOR(int, unlink, char *path) { SCOPED_TSAN_INTERCEPTOR(unlink, path); Release(thr, pc, File2addr(path)); @@ -1814,12 +1693,30 @@ TSAN_INTERCEPTOR(int, rmdir, char *path) { TSAN_INTERCEPTOR(int, closedir, void *dirp) { SCOPED_TSAN_INTERCEPTOR(closedir, dirp); - int fd = dirfd(dirp); - FdClose(thr, pc, fd); + if (dirp) { + int fd = dirfd(dirp); + FdClose(thr, pc, fd); + } return REAL(closedir)(dirp); } #if SANITIZER_LINUX +TSAN_INTERCEPTOR(int, epoll_create, int size) { + SCOPED_TSAN_INTERCEPTOR(epoll_create, size); + int fd = REAL(epoll_create)(size); + if (fd >= 0) + FdPollCreate(thr, pc, fd); + return fd; +} + +TSAN_INTERCEPTOR(int, epoll_create1, int flags) { + SCOPED_TSAN_INTERCEPTOR(epoll_create1, flags); + int fd = REAL(epoll_create1)(flags); + if (fd >= 0) + FdPollCreate(thr, pc, fd); + return fd; +} + TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) { SCOPED_TSAN_INTERCEPTOR(epoll_ctl, epfd, op, fd, ev); if (epfd >= 0) @@ -1831,12 +1728,7 @@ TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) { int res = REAL(epoll_ctl)(epfd, op, fd, ev); return res; } -#define TSAN_MAYBE_INTERCEPT_EPOLL_CTL TSAN_INTERCEPT(epoll_ctl) -#else -#define TSAN_MAYBE_INTERCEPT_EPOLL_CTL -#endif -#if SANITIZER_LINUX TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) { SCOPED_TSAN_INTERCEPTOR(epoll_wait, epfd, ev, cnt, timeout); if (epfd >= 0) @@ -1846,17 +1738,72 @@ TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) { FdAcquire(thr, pc, epfd); return res; } -#define TSAN_MAYBE_INTERCEPT_EPOLL_WAIT TSAN_INTERCEPT(epoll_wait) + +TSAN_INTERCEPTOR(int, epoll_pwait, int epfd, void *ev, int cnt, int timeout, + void *sigmask) { + SCOPED_TSAN_INTERCEPTOR(epoll_pwait, epfd, ev, cnt, timeout, sigmask); + if (epfd >= 0) + FdAccess(thr, pc, epfd); + int res = BLOCK_REAL(epoll_pwait)(epfd, ev, cnt, timeout, sigmask); + if (res > 0 && epfd >= 0) + FdAcquire(thr, pc, epfd); + return res; +} + +#define TSAN_MAYBE_INTERCEPT_EPOLL \ + TSAN_INTERCEPT(epoll_create); \ + TSAN_INTERCEPT(epoll_create1); \ + TSAN_INTERCEPT(epoll_ctl); \ + TSAN_INTERCEPT(epoll_wait); \ + TSAN_INTERCEPT(epoll_pwait) #else -#define TSAN_MAYBE_INTERCEPT_EPOLL_WAIT +#define TSAN_MAYBE_INTERCEPT_EPOLL #endif +// The following functions are intercepted merely to process pending signals. +// If program blocks signal X, we must deliver the signal before the function +// returns. Similarly, if program unblocks a signal (or returns from sigsuspend) +// it's better to deliver the signal straight away. +TSAN_INTERCEPTOR(int, sigsuspend, const __sanitizer_sigset_t *mask) { + SCOPED_TSAN_INTERCEPTOR(sigsuspend, mask); + return REAL(sigsuspend)(mask); +} + +TSAN_INTERCEPTOR(int, sigblock, int mask) { + SCOPED_TSAN_INTERCEPTOR(sigblock, mask); + return REAL(sigblock)(mask); +} + +TSAN_INTERCEPTOR(int, sigsetmask, int mask) { + SCOPED_TSAN_INTERCEPTOR(sigsetmask, mask); + return REAL(sigsetmask)(mask); +} + +TSAN_INTERCEPTOR(int, pthread_sigmask, int how, const __sanitizer_sigset_t *set, + __sanitizer_sigset_t *oldset) { + SCOPED_TSAN_INTERCEPTOR(pthread_sigmask, how, set, oldset); + return REAL(pthread_sigmask)(how, set, oldset); +} + namespace __tsan { static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire, bool sigact, int sig, my_siginfo_t *info, void *uctx) { if (acquire) Acquire(thr, 0, (uptr)&sigactions[sig]); + // Signals are generally asynchronous, so if we receive a signals when + // ignores are enabled we should disable ignores. This is critical for sync + // and interceptors, because otherwise we can miss syncronization and report + // false races. + int ignore_reads_and_writes = thr->ignore_reads_and_writes; + int ignore_interceptors = thr->ignore_interceptors; + int ignore_sync = thr->ignore_sync; + if (!ctx->after_multithreaded_fork) { + thr->ignore_reads_and_writes = 0; + thr->fast_state.ClearIgnoreBit(); + thr->ignore_interceptors = 0; + thr->ignore_sync = 0; + } // Ensure that the handler does not spoil errno. const int saved_errno = errno; errno = 99; @@ -1872,6 +1819,13 @@ static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire, else ((sighandler_t)pc)(sig); } + if (!ctx->after_multithreaded_fork) { + thr->ignore_reads_and_writes = ignore_reads_and_writes; + if (ignore_reads_and_writes) + thr->fast_state.SetIgnoreBit(); + thr->ignore_interceptors = ignore_interceptors; + thr->ignore_sync = ignore_sync; + } // We do not detect errno spoiling for SIGTERM, // because some SIGTERM handlers do spoil errno but reraise SIGTERM, // tsan reports false positive in such case. @@ -1901,10 +1855,9 @@ void ProcessPendingSignals(ThreadState *thr) { return; atomic_store(&sctx->have_pending_signals, 0, memory_order_relaxed); atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed); - // These are too big for stack. - static THREADLOCAL __sanitizer_sigset_t emptyset, oldset; - CHECK_EQ(0, REAL(sigfillset)(&emptyset)); - CHECK_EQ(0, pthread_sigmask(SIG_SETMASK, &emptyset, &oldset)); + internal_sigfillset(&sctx->emptyset); + int res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->emptyset, &sctx->oldset); + CHECK_EQ(res, 0); for (int sig = 0; sig < kSigCount; sig++) { SignalDesc *signal = &sctx->pending_signals[sig]; if (signal->armed) { @@ -1913,7 +1866,8 @@ void ProcessPendingSignals(ThreadState *thr) { &signal->siginfo, &signal->ctx); } } - CHECK_EQ(0, pthread_sigmask(SIG_SETMASK, &oldset, 0)); + res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->oldset, 0); + CHECK_EQ(res, 0); atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed); } @@ -1943,13 +1897,8 @@ void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig, (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed))) { atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed); if (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed)) { - // We ignore interceptors in blocking functions, - // temporary enbled them again while we are calling user function. - int const i = thr->ignore_interceptors; - thr->ignore_interceptors = 0; atomic_store(&sctx->in_blocking_func, 0, memory_order_relaxed); CallUserSignalHandler(thr, sync, true, sigact, sig, info, ctx); - thr->ignore_interceptors = i; atomic_store(&sctx->in_blocking_func, 1, memory_order_relaxed); } else { // Be very conservative with when we do acquire in this case. @@ -1987,7 +1936,10 @@ static void rtl_sigaction(int sig, my_siginfo_t *info, void *ctx) { } TSAN_INTERCEPTOR(int, sigaction, int sig, sigaction_t *act, sigaction_t *old) { - SCOPED_TSAN_INTERCEPTOR(sigaction, sig, act, old); + // Note: if we call REAL(sigaction) directly for any reason without proxying + // the signal handler through rtl_sigaction, very bad things will happen. + // The handler will run synchronously and corrupt tsan per-thread state. + SCOPED_INTERCEPTOR_RAW(sigaction, sig, act, old); if (old) internal_memcpy(old, &sigactions[sig], sizeof(*old)); if (act == 0) @@ -2002,12 +1954,12 @@ TSAN_INTERCEPTOR(int, sigaction, int sig, sigaction_t *act, sigaction_t *old) { sigactions[sig].sa_flags = *(volatile int*)&act->sa_flags; internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask, sizeof(sigactions[sig].sa_mask)); -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_MAC sigactions[sig].sa_restorer = act->sa_restorer; #endif sigaction_t newact; internal_memcpy(&newact, act, sizeof(newact)); - REAL(sigfillset)(&newact.sa_mask); + internal_sigfillset(&newact.sa_mask); if (act->sa_handler != SIG_IGN && act->sa_handler != SIG_DFL) { if (newact.sa_flags & SA_SIGINFO) newact.sa_sigaction = rtl_sigaction; @@ -2022,7 +1974,7 @@ TSAN_INTERCEPTOR(int, sigaction, int sig, sigaction_t *act, sigaction_t *old) { TSAN_INTERCEPTOR(sighandler_t, signal, int sig, sighandler_t h) { sigaction_t act; act.sa_handler = h; - REAL(memset)(&act.sa_mask, -1, sizeof(act.sa_mask)); + internal_memset(&act.sa_mask, -1, sizeof(act.sa_mask)); act.sa_flags = 0; sigaction_t old; int res = sigaction(sig, &act, &old); @@ -2031,11 +1983,6 @@ TSAN_INTERCEPTOR(sighandler_t, signal, int sig, sighandler_t h) { return old.sa_handler; } -TSAN_INTERCEPTOR(int, sigsuspend, const __sanitizer_sigset_t *mask) { - SCOPED_TSAN_INTERCEPTOR(sigsuspend, mask); - return REAL(sigsuspend)(mask); -} - TSAN_INTERCEPTOR(int, raise, int sig) { SCOPED_TSAN_INTERCEPTOR(raise, sig); ThreadSignalContext *sctx = SigCtx(thr); @@ -2103,7 +2050,13 @@ TSAN_INTERCEPTOR(int, fork, int fake) { return REAL(fork)(fake); SCOPED_INTERCEPTOR_RAW(fork, fake); ForkBefore(thr, pc); - int pid = REAL(fork)(fake); + int pid; + { + // On OS X, REAL(fork) can call intercepted functions (OSSpinLockLock), and + // we'll assert in CheckNoLocks() unless we ignore interceptors. + ScopedIgnoreInterceptors ignore; + pid = REAL(fork)(fake); + } if (pid == 0) { // child ForkChildAfter(thr, pc); @@ -2135,7 +2088,7 @@ TSAN_INTERCEPTOR(int, vfork, int fake) { return WRAP(fork)(fake); } -#if !SANITIZER_MAC +#if !SANITIZER_MAC && !SANITIZER_ANDROID typedef int (*dl_iterate_phdr_cb_t)(__sanitizer_dl_phdr_info *info, SIZE_T size, void *data); struct dl_iterate_phdr_data { @@ -2218,23 +2171,15 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, #undef SANITIZER_INTERCEPT_FGETPWENT #undef SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS #undef SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS -// __tls_get_addr can be called with mis-aligned stack due to: -// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58066 -// There are two potential issues: -// 1. Sanitizer code contains a MOVDQA spill (it does not seem to be the case -// right now). or 2. ProcessPendingSignal calls user handler which contains -// MOVDQA spill (this happens right now). -// Since the interceptor only initializes memory for msan, the simplest solution -// is to disable the interceptor in tsan (other sanitizers do not call -// signal handlers from COMMON_INTERCEPTOR_ENTER). -// As __tls_get_addr has been intercepted in the past, to avoid breaking -// libtsan ABI, keep it around, but just call the real function. +// We define our own. #if SANITIZER_INTERCEPT_TLS_GET_ADDR #define NEED_TLS_GET_ADDR #endif #undef SANITIZER_INTERCEPT_TLS_GET_ADDR #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) +#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \ + INTERCEPT_FUNCTION_VER(name, ver) #define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr, \ @@ -2321,6 +2266,10 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, MutexRepair(((TsanInterceptorContext *)ctx)->thr, \ ((TsanInterceptorContext *)ctx)->pc, (uptr)m) +#define COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m) \ + MutexInvalidAccess(((TsanInterceptorContext *)ctx)->thr, \ + ((TsanInterceptorContext *)ctx)->pc, (uptr)m) + #if !SANITIZER_MAC #define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \ HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \ @@ -2335,6 +2284,12 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, *begin = *end = 0; \ } +#define COMMON_INTERCEPTOR_USER_CALLBACK_START() \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START() + +#define COMMON_INTERCEPTOR_USER_CALLBACK_END() \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END() + #include "sanitizer_common/sanitizer_common_interceptors.inc" #define TSAN_SYSCALL() \ @@ -2357,7 +2312,7 @@ struct ScopedSyscall { } }; -#if !SANITIZER_MAC +#if !SANITIZER_FREEBSD && !SANITIZER_MAC static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) { TSAN_SYSCALL(); MemoryAccessRange(thr, pc, p, s, write); @@ -2452,8 +2407,27 @@ static void syscall_post_fork(uptr pc, int pid) { #include "sanitizer_common/sanitizer_common_syscalls.inc" #ifdef NEED_TLS_GET_ADDR +// Define own interceptor instead of sanitizer_common's for three reasons: +// 1. It must not process pending signals. +// Signal handlers may contain MOVDQA instruction (see below). +// 2. It must be as simple as possible to not contain MOVDQA. +// 3. Sanitizer_common version uses COMMON_INTERCEPTOR_INITIALIZE_RANGE which +// is empty for tsan (meant only for msan). +// Note: __tls_get_addr can be called with mis-aligned stack due to: +// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58066 +// So the interceptor must work with mis-aligned stack, in particular, does not +// execute MOVDQA with stack addresses. TSAN_INTERCEPTOR(void *, __tls_get_addr, void *arg) { - return REAL(__tls_get_addr)(arg); + void *res = REAL(__tls_get_addr)(arg); + ThreadState *thr = cur_thread(); + if (!thr) + return res; + DTLS::DTV *dtv = DTLS_on_tls_get_addr(arg, res, thr->tls_addr, thr->tls_size); + if (!dtv) + return res; + // New DTLS block has been allocated. + MemoryResetRange(thr, 0, dtv->beg, dtv->size); + return res; } #endif @@ -2468,7 +2442,7 @@ static void finalize(void *arg) { Die(); } -#if !SANITIZER_MAC +#if !SANITIZER_MAC && !SANITIZER_ANDROID static void unreachable() { Report("FATAL: ThreadSanitizer: unreachable called\n"); Die(); @@ -2517,13 +2491,6 @@ void InitializeInterceptors() { TSAN_MAYBE_INTERCEPT_PVALLOC; TSAN_INTERCEPT(posix_memalign); - TSAN_INTERCEPT(strlen); - TSAN_INTERCEPT(memset); - TSAN_INTERCEPT(memcpy); - TSAN_INTERCEPT(memmove); - TSAN_INTERCEPT(strchr); - TSAN_INTERCEPT(strchrnul); - TSAN_INTERCEPT(strrchr); TSAN_INTERCEPT(strcpy); // NOLINT TSAN_INTERCEPT(strncpy); TSAN_INTERCEPT(strdup); @@ -2566,14 +2533,6 @@ void InitializeInterceptors() { TSAN_INTERCEPT(pthread_once); - TSAN_INTERCEPT(stat); - TSAN_MAYBE_INTERCEPT___XSTAT; - TSAN_MAYBE_INTERCEPT_STAT64; - TSAN_MAYBE_INTERCEPT___XSTAT64; - TSAN_INTERCEPT(lstat); - TSAN_MAYBE_INTERCEPT___LXSTAT; - TSAN_MAYBE_INTERCEPT_LSTAT64; - TSAN_MAYBE_INTERCEPT___LXSTAT64; TSAN_INTERCEPT(fstat); TSAN_MAYBE_INTERCEPT___FXSTAT; TSAN_MAYBE_INTERCEPT_FSTAT64; @@ -2594,18 +2553,13 @@ void InitializeInterceptors() { TSAN_INTERCEPT(connect); TSAN_INTERCEPT(bind); TSAN_INTERCEPT(listen); - TSAN_MAYBE_INTERCEPT_EPOLL_CREATE; - TSAN_MAYBE_INTERCEPT_EPOLL_CREATE1; + TSAN_MAYBE_INTERCEPT_EPOLL; TSAN_INTERCEPT(close); TSAN_MAYBE_INTERCEPT___CLOSE; TSAN_MAYBE_INTERCEPT___RES_ICLOSE; TSAN_INTERCEPT(pipe); TSAN_INTERCEPT(pipe2); - TSAN_INTERCEPT(send); - TSAN_INTERCEPT(sendmsg); - TSAN_INTERCEPT(recv); - TSAN_INTERCEPT(unlink); TSAN_INTERCEPT(tmpfile); TSAN_MAYBE_INTERCEPT_TMPFILE64; @@ -2616,12 +2570,12 @@ void InitializeInterceptors() { TSAN_INTERCEPT(rmdir); TSAN_INTERCEPT(closedir); - TSAN_MAYBE_INTERCEPT_EPOLL_CTL; - TSAN_MAYBE_INTERCEPT_EPOLL_WAIT; - TSAN_INTERCEPT(sigaction); TSAN_INTERCEPT(signal); TSAN_INTERCEPT(sigsuspend); + TSAN_INTERCEPT(sigblock); + TSAN_INTERCEPT(sigsetmask); + TSAN_INTERCEPT(pthread_sigmask); TSAN_INTERCEPT(raise); TSAN_INTERCEPT(kill); TSAN_INTERCEPT(pthread_kill); @@ -2633,7 +2587,9 @@ void InitializeInterceptors() { TSAN_INTERCEPT(fork); TSAN_INTERCEPT(vfork); +#if !SANITIZER_ANDROID TSAN_INTERCEPT(dl_iterate_phdr); +#endif TSAN_INTERCEPT(on_exit); TSAN_INTERCEPT(__cxa_atexit); TSAN_INTERCEPT(_exit); @@ -2642,7 +2598,7 @@ void InitializeInterceptors() { TSAN_INTERCEPT(__tls_get_addr); #endif -#if !SANITIZER_MAC +#if !SANITIZER_MAC && !SANITIZER_ANDROID // Need to setup it, because interceptors check that the function is resolved. // But atexit is emitted directly into the module, so can't be resolved. REAL(atexit) = (int(*)(void(*)()))unreachable; @@ -2653,12 +2609,50 @@ void InitializeInterceptors() { Die(); } +#if !SANITIZER_MAC if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) { Printf("ThreadSanitizer: failed to create thread key\n"); Die(); } +#endif FdInit(); } } // namespace __tsan + +// Invisible barrier for tests. +// There were several unsuccessful iterations for this functionality: +// 1. Initially it was implemented in user code using +// REAL(pthread_barrier_wait). But pthread_barrier_wait is not supported on +// MacOS. Futexes are linux-specific for this matter. +// 2. Then we switched to atomics+usleep(10). But usleep produced parasitic +// "as-if synchronized via sleep" messages in reports which failed some +// output tests. +// 3. Then we switched to atomics+sched_yield. But this produced tons of tsan- +// visible events, which lead to "failed to restore stack trace" failures. +// Note that no_sanitize_thread attribute does not turn off atomic interception +// so attaching it to the function defined in user code does not help. +// That's why we now have what we have. +extern "C" SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_testonly_barrier_init(u64 *barrier, u32 count) { + if (count >= (1 << 8)) { + Printf("barrier_init: count is too large (%d)\n", count); + Die(); + } + // 8 lsb is thread count, the remaining are count of entered threads. + *barrier = count; +} + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_testonly_barrier_wait(u64 *barrier) { + unsigned old = __atomic_fetch_add(barrier, 1 << 8, __ATOMIC_RELAXED); + unsigned old_epoch = (old >> 8) / (old & 0xff); + for (;;) { + unsigned cur = __atomic_load_n(barrier, __ATOMIC_RELAXED); + unsigned cur_epoch = (cur >> 8) / (cur & 0xff); + if (cur_epoch != old_epoch) + return; + internal_sched_yield(); + } +} diff --git a/libsanitizer/tsan/tsan_interceptors.h b/libsanitizer/tsan/tsan_interceptors.h index ed68eb96bf81..97fa5085a789 100644 --- a/libsanitizer/tsan/tsan_interceptors.h +++ b/libsanitizer/tsan/tsan_interceptors.h @@ -8,6 +8,8 @@ class ScopedInterceptor { public: ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc); ~ScopedInterceptor(); + void UserCallbackStart(); + void UserCallbackEnd(); private: ThreadState *const thr_; const uptr pc_; @@ -30,18 +32,16 @@ class ScopedInterceptor { Report("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \ Die(); \ } \ - if (thr->ignore_interceptors || thr->in_ignored_lib) \ + if (!thr->is_inited || thr->ignore_interceptors || thr->in_ignored_lib) \ return REAL(func)(__VA_ARGS__); \ /**/ -#define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__) +#define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START() \ + si.UserCallbackStart(); -#if SANITIZER_FREEBSD -#define __libc_free __free -#define __libc_malloc __malloc -#endif +#define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END() \ + si.UserCallbackEnd(); -extern "C" void __libc_free(void *ptr); -extern "C" void *__libc_malloc(uptr size); +#define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__) #endif // TSAN_INTERCEPTORS_H diff --git a/libsanitizer/tsan/tsan_interceptors_mac.cc b/libsanitizer/tsan/tsan_interceptors_mac.cc new file mode 100644 index 000000000000..eaf866d6c75c --- /dev/null +++ b/libsanitizer/tsan/tsan_interceptors_mac.cc @@ -0,0 +1,357 @@ +//===-- tsan_interceptors_mac.cc ------------------------------------------===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// Mac-specific interceptors. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#if SANITIZER_MAC + +#include "interception/interception.h" +#include "tsan_interceptors.h" +#include "tsan_interface.h" +#include "tsan_interface_ann.h" + +#include <libkern/OSAtomic.h> +#include <xpc/xpc.h> + +typedef long long_t; // NOLINT + +namespace __tsan { + +// The non-barrier versions of OSAtomic* functions are semantically mo_relaxed, +// but the two variants (e.g. OSAtomicAdd32 and OSAtomicAdd32Barrier) are +// actually aliases of each other, and we cannot have different interceptors for +// them, because they're actually the same function. Thus, we have to stay +// conservative and treat the non-barrier versions as mo_acq_rel. +static const morder kMacOrderBarrier = mo_acq_rel; +static const morder kMacOrderNonBarrier = mo_acq_rel; + +#define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ + TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \ + return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \ + } + +#define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ + TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \ + return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \ + } + +#define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ + TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f, ptr); \ + return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \ + } + +#define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \ + mo) \ + TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f, ptr); \ + return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \ + } + +#define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \ + m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \ + kMacOrderNonBarrier) \ + m(int32_t, int32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \ + kMacOrderBarrier) \ + m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \ + kMacOrderNonBarrier) \ + m(int64_t, int64_t, a64, f##64##Barrier, __tsan_atomic64_##tsan_atomic_f, \ + kMacOrderBarrier) + +#define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \ + m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \ + kMacOrderNonBarrier) \ + m(int32_t, uint32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \ + kMacOrderBarrier) \ + m_orig(int32_t, uint32_t, a32, f##32##Orig, __tsan_atomic32_##tsan_atomic_f, \ + kMacOrderNonBarrier) \ + m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \ + __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier) + +OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add, + OSATOMIC_INTERCEPTOR_PLUS_X) +OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add, + OSATOMIC_INTERCEPTOR_PLUS_1) +OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicDecrement, fetch_sub, + OSATOMIC_INTERCEPTOR_MINUS_1) +OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicOr, fetch_or, OSATOMIC_INTERCEPTOR_PLUS_X, + OSATOMIC_INTERCEPTOR) +OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and, + OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR) +OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor, + OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR) + +#define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \ + TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \ + return tsan_atomic_f##_compare_exchange_strong( \ + (tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ + kMacOrderNonBarrier, kMacOrderNonBarrier); \ + } \ + \ + TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \ + t volatile *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \ + return tsan_atomic_f##_compare_exchange_strong( \ + (tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ + kMacOrderBarrier, kMacOrderNonBarrier); \ + } + +OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int) +OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64, + long_t) +OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapPtr, __tsan_atomic64, a64, + void *) +OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32, + int32_t) +OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64, + int64_t) + +#define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \ + TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \ + char *byte_ptr = ((char *)ptr) + (n >> 3); \ + char bit = 0x80u >> (n & 7); \ + char mask = clear ? ~bit : bit; \ + char orig_byte = op((a8 *)byte_ptr, mask, mo); \ + return orig_byte & bit; \ + } + +#define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \ + OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \ + OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier) + +OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false) +OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and, + true) + +TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead *list, void *item, + size_t offset) { + SCOPED_TSAN_INTERCEPTOR(OSAtomicEnqueue, list, item, offset); + __tsan_release(item); + REAL(OSAtomicEnqueue)(list, item, offset); +} + +TSAN_INTERCEPTOR(void *, OSAtomicDequeue, OSQueueHead *list, size_t offset) { + SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset); + void *item = REAL(OSAtomicDequeue)(list, offset); + if (item) __tsan_acquire(item); + return item; +} + +// OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X. +#if !SANITIZER_IOS + +TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead *list, void *item, + size_t offset) { + SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoEnqueue, list, item, offset); + __tsan_release(item); + REAL(OSAtomicFifoEnqueue)(list, item, offset); +} + +TSAN_INTERCEPTOR(void *, OSAtomicFifoDequeue, OSFifoQueueHead *list, + size_t offset) { + SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset); + void *item = REAL(OSAtomicFifoDequeue)(list, offset); + if (item) __tsan_acquire(item); + return item; +} + +#endif + +TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) { + CHECK(!cur_thread()->is_dead); + if (!cur_thread()->is_inited) { + return REAL(OSSpinLockLock)(lock); + } + SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock, lock); + REAL(OSSpinLockLock)(lock); + Acquire(thr, pc, (uptr)lock); +} + +TSAN_INTERCEPTOR(bool, OSSpinLockTry, volatile OSSpinLock *lock) { + CHECK(!cur_thread()->is_dead); + if (!cur_thread()->is_inited) { + return REAL(OSSpinLockTry)(lock); + } + SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry, lock); + bool result = REAL(OSSpinLockTry)(lock); + if (result) + Acquire(thr, pc, (uptr)lock); + return result; +} + +TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) { + CHECK(!cur_thread()->is_dead); + if (!cur_thread()->is_inited) { + return REAL(OSSpinLockUnlock)(lock); + } + SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock, lock); + Release(thr, pc, (uptr)lock); + REAL(OSSpinLockUnlock)(lock); +} + +TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) { + CHECK(!cur_thread()->is_dead); + if (!cur_thread()->is_inited) { + return REAL(os_lock_lock)(lock); + } + SCOPED_TSAN_INTERCEPTOR(os_lock_lock, lock); + REAL(os_lock_lock)(lock); + Acquire(thr, pc, (uptr)lock); +} + +TSAN_INTERCEPTOR(bool, os_lock_trylock, void *lock) { + CHECK(!cur_thread()->is_dead); + if (!cur_thread()->is_inited) { + return REAL(os_lock_trylock)(lock); + } + SCOPED_TSAN_INTERCEPTOR(os_lock_trylock, lock); + bool result = REAL(os_lock_trylock)(lock); + if (result) + Acquire(thr, pc, (uptr)lock); + return result; +} + +TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) { + CHECK(!cur_thread()->is_dead); + if (!cur_thread()->is_inited) { + return REAL(os_lock_unlock)(lock); + } + SCOPED_TSAN_INTERCEPTOR(os_lock_unlock, lock); + Release(thr, pc, (uptr)lock); + REAL(os_lock_unlock)(lock); +} + +TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler, + xpc_connection_t connection, xpc_handler_t handler) { + SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection, + handler); + Release(thr, pc, (uptr)connection); + xpc_handler_t new_handler = ^(xpc_object_t object) { + { + SCOPED_INTERCEPTOR_RAW(xpc_connection_set_event_handler); + Acquire(thr, pc, (uptr)connection); + } + handler(object); + }; + REAL(xpc_connection_set_event_handler)(connection, new_handler); +} + +TSAN_INTERCEPTOR(void, xpc_connection_send_barrier, xpc_connection_t connection, + dispatch_block_t barrier) { + SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_barrier, connection, barrier); + Release(thr, pc, (uptr)connection); + dispatch_block_t new_barrier = ^() { + { + SCOPED_INTERCEPTOR_RAW(xpc_connection_send_barrier); + Acquire(thr, pc, (uptr)connection); + } + barrier(); + }; + REAL(xpc_connection_send_barrier)(connection, new_barrier); +} + +TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply, + xpc_connection_t connection, xpc_object_t message, + dispatch_queue_t replyq, xpc_handler_t handler) { + SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_message_with_reply, connection, + message, replyq, handler); + Release(thr, pc, (uptr)connection); + xpc_handler_t new_handler = ^(xpc_object_t object) { + { + SCOPED_INTERCEPTOR_RAW(xpc_connection_send_message_with_reply); + Acquire(thr, pc, (uptr)connection); + } + handler(object); + }; + REAL(xpc_connection_send_message_with_reply) + (connection, message, replyq, new_handler); +} + +// On macOS, libc++ is always linked dynamically, so intercepting works the +// usual way. +#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR + +namespace { +struct fake_shared_weak_count { + volatile a64 shared_owners; + volatile a64 shared_weak_owners; + virtual void _unused_0x0() = 0; + virtual void _unused_0x8() = 0; + virtual void on_zero_shared() = 0; + virtual void _unused_0x18() = 0; + virtual void on_zero_shared_weak() = 0; +}; +} // namespace + +// This adds a libc++ interceptor for: +// void __shared_weak_count::__release_shared() _NOEXCEPT; +// Shared and weak pointers in C++ maintain reference counts via atomics in +// libc++.dylib, which are TSan-invisible, and this leads to false positives in +// destructor code. This interceptor re-implements the whole function so that +// the mo_acq_rel semantics of the atomic decrement are visible. +// +// Unfortunately, this interceptor cannot simply Acquire/Release some sync +// object and call the original function, because it would have a race between +// the sync and the destruction of the object. Calling both under a lock will +// not work because the destructor can invoke this interceptor again (and even +// in a different thread, so recursive locks don't help). +STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv, + fake_shared_weak_count *o) { + if (!flags()->shared_ptr_interceptor) + return REAL(_ZNSt3__119__shared_weak_count16__release_sharedEv)(o); + + SCOPED_TSAN_INTERCEPTOR(_ZNSt3__119__shared_weak_count16__release_sharedEv, + o); + if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) { + Acquire(thr, pc, (uptr)&o->shared_owners); + o->on_zero_shared(); + if (__tsan_atomic64_fetch_add(&o->shared_weak_owners, -1, mo_release) == + 0) { + Acquire(thr, pc, (uptr)&o->shared_weak_owners); + o->on_zero_shared_weak(); + } + } +} + +namespace { +struct call_once_callback_args { + void (*orig_func)(void *arg); + void *orig_arg; + void *flag; +}; + +void call_once_callback_wrapper(void *arg) { + call_once_callback_args *new_args = (call_once_callback_args *)arg; + new_args->orig_func(new_args->orig_arg); + __tsan_release(new_args->flag); +} +} // namespace + +// This adds a libc++ interceptor for: +// void __call_once(volatile unsigned long&, void*, void(*)(void*)); +// C++11 call_once is implemented via an internal function __call_once which is +// inside libc++.dylib, and the atomic release store inside it is thus +// TSan-invisible. To avoid false positives, this interceptor wraps the callback +// function and performs an explicit Release after the user code has run. +STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag, + void *arg, void (*func)(void *arg)) { + call_once_callback_args new_args = {func, arg, flag}; + REAL(_ZNSt3__111__call_onceERVmPvPFvS2_E)(flag, &new_args, + call_once_callback_wrapper); +} + +} // namespace __tsan + +#endif // SANITIZER_MAC diff --git a/libsanitizer/tsan/tsan_interface.h b/libsanitizer/tsan/tsan_interface.h index 4ff6fa1831ee..066dde6f5432 100644 --- a/libsanitizer/tsan/tsan_interface.h +++ b/libsanitizer/tsan/tsan_interface.h @@ -15,6 +15,7 @@ #define TSAN_INTERFACE_H #include <sanitizer_common/sanitizer_internal_defs.h> +using __sanitizer::uptr; // This header should NOT include any other headers. // All functions in this header are extern "C" and start with __tsan_. @@ -23,6 +24,8 @@ extern "C" { #endif +#if !SANITIZER_GO + // This function should be called at the very beginning of the process, // before any instrumented code is executed and before any call to malloc. SANITIZER_INTERFACE_ATTRIBUTE void __tsan_init(); @@ -73,8 +76,296 @@ void __tsan_read_range(void *addr, unsigned long size); // NOLINT SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write_range(void *addr, unsigned long size); // NOLINT +// User may provide function that would be called right when TSan detects +// an error. The argument 'report' is an opaque pointer that can be used to +// gather additional information using other TSan report API functions. +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_on_report(void *report); + +// If TSan is currently reporting a detected issue on the current thread, +// returns an opaque pointer to the current report. Otherwise returns NULL. +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_get_current_report(); + +// Returns a report's description (issue type), number of duplicate issues +// found, counts of array data (stack traces, memory operations, locations, +// mutexes, threads, unique thread IDs) and a stack trace of a sleep() call (if +// one was involved in the issue). +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_data(void *report, const char **description, int *count, + int *stack_count, int *mop_count, int *loc_count, + int *mutex_count, int *thread_count, + int *unique_tid_count, void **sleep_trace, + uptr trace_size); + +// Returns information about stack traces included in the report. +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_stack(void *report, uptr idx, void **trace, + uptr trace_size); + +// Returns information about memory operations included in the report. +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_mop(void *report, uptr idx, int *tid, void **addr, + int *size, int *write, int *atomic, void **trace, + uptr trace_size); + +// Returns information about locations included in the report. +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_loc(void *report, uptr idx, const char **type, + void **addr, uptr *start, uptr *size, int *tid, + int *fd, int *suppressable, void **trace, + uptr trace_size); + +// Returns information about mutexes included in the report. +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_mutex(void *report, uptr idx, uptr *mutex_id, void **addr, + int *destroyed, void **trace, uptr trace_size); + +// Returns information about threads included in the report. +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_thread(void *report, uptr idx, int *tid, uptr *os_id, + int *running, const char **name, int *parent_tid, + void **trace, uptr trace_size); + +// Returns information about unique thread IDs included in the report. +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_get_report_unique_tid(void *report, uptr idx, int *tid); + +#endif // SANITIZER_GO + #ifdef __cplusplus } // extern "C" #endif +namespace __tsan { + +// These should match declarations from public tsan_interface_atomic.h header. +typedef unsigned char a8; +typedef unsigned short a16; // NOLINT +typedef unsigned int a32; +typedef unsigned long long a64; // NOLINT +#if !SANITIZER_GO && (defined(__SIZEOF_INT128__) \ + || (__clang_major__ * 100 + __clang_minor__ >= 302)) && !defined(__mips64) +__extension__ typedef __int128 a128; +# define __TSAN_HAS_INT128 1 +#else +# define __TSAN_HAS_INT128 0 +#endif + +// Part of ABI, do not change. +// http://llvm.org/viewvc/llvm-project/libcxx/trunk/include/atomic?view=markup +typedef enum { + mo_relaxed, + mo_consume, + mo_acquire, + mo_release, + mo_acq_rel, + mo_seq_cst +} morder; + +struct ThreadState; + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_load(const volatile a8 *a, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_load(const volatile a16 *a, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_load(const volatile a32 *a, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_load(const volatile a64 *a, morder mo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_load(const volatile a128 *a, morder mo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v, + morder mo, morder fmo); +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v, + morder mo, morder fmo); +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v, + morder mo, morder fmo); +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v, + morder mo, morder fmo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v, + morder mo, morder fmo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v, morder mo, + morder fmo); +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v, + morder mo, morder fmo); +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v, + morder mo, morder fmo); +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v, + morder mo, morder fmo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v, + morder mo, morder fmo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, morder mo, + morder fmo); +SANITIZER_INTERFACE_ATTRIBUTE +a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v, + morder mo, morder fmo); +SANITIZER_INTERFACE_ATTRIBUTE +a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v, + morder mo, morder fmo); +SANITIZER_INTERFACE_ATTRIBUTE +a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v, + morder mo, morder fmo); +#if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE +a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v, + morder mo, morder fmo); +#endif + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_atomic_thread_fence(morder mo); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_atomic_signal_fence(morder mo); + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_compare_exchange(ThreadState *thr, uptr cpc, uptr pc, + u8 *a); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_compare_exchange(ThreadState *thr, uptr cpc, uptr pc, + u8 *a); +} // extern "C" + +} // namespace __tsan + #endif // TSAN_INTERFACE_H diff --git a/libsanitizer/tsan/tsan_interface_atomic.cc b/libsanitizer/tsan/tsan_interface_atomic.cc index 0ded1aa10991..5c5c34f3b876 100644 --- a/libsanitizer/tsan/tsan_interface_atomic.cc +++ b/libsanitizer/tsan/tsan_interface_atomic.cc @@ -21,39 +21,16 @@ #include "sanitizer_common/sanitizer_stacktrace.h" #include "sanitizer_common/sanitizer_mutex.h" #include "tsan_flags.h" +#include "tsan_interface.h" #include "tsan_rtl.h" using namespace __tsan; // NOLINT -// These should match declarations from public tsan_interface_atomic.h header. -typedef unsigned char a8; -typedef unsigned short a16; // NOLINT -typedef unsigned int a32; -typedef unsigned long long a64; // NOLINT -#if !defined(SANITIZER_GO) && (defined(__SIZEOF_INT128__) \ - || (__clang_major__ * 100 + __clang_minor__ >= 302)) && !defined(__mips64) -__extension__ typedef __int128 a128; -# define __TSAN_HAS_INT128 1 -#else -# define __TSAN_HAS_INT128 0 -#endif - -#if !defined(SANITIZER_GO) && __TSAN_HAS_INT128 +#if !SANITIZER_GO && __TSAN_HAS_INT128 // Protects emulation of 128-bit atomic operations. static StaticSpinMutex mutex128; #endif -// Part of ABI, do not change. -// http://llvm.org/viewvc/llvm-project/libcxx/trunk/include/atomic?view=markup -typedef enum { - mo_relaxed, - mo_consume, - mo_acquire, - mo_release, - mo_acq_rel, - mo_seq_cst -} morder; - static bool IsLoadOrder(morder mo) { return mo == mo_relaxed || mo == mo_consume || mo == mo_acquire || mo == mo_seq_cst; @@ -123,7 +100,7 @@ template<typename T> T func_cas(volatile T *v, T cmp, T xch) { // Atomic ops are executed under tsan internal mutex, // here we assume that the atomic variables are not accessed // from non-instrumented code. -#if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !defined(SANITIZER_GO) \ +#if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !SANITIZER_GO \ && __TSAN_HAS_INT128 a128 func_xchg(volatile a128 *v, a128 op) { SpinMutexLock lock(&mutex128); @@ -197,7 +174,7 @@ static int SizeLog() { // this leads to false negatives only in very obscure cases. } -#ifndef SANITIZER_GO +#if !SANITIZER_GO static atomic_uint8_t *to_atomic(const volatile a8 *a) { return reinterpret_cast<atomic_uint8_t *>(const_cast<a8 *>(a)); } @@ -233,7 +210,7 @@ static T NoTsanAtomicLoad(const volatile T *a, morder mo) { return atomic_load(to_atomic(a), to_mo(mo)); } -#if __TSAN_HAS_INT128 && !defined(SANITIZER_GO) +#if __TSAN_HAS_INT128 && !SANITIZER_GO static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) { SpinMutexLock lock(&mutex128); return *a; @@ -263,7 +240,7 @@ static void NoTsanAtomicStore(volatile T *a, T v, morder mo) { atomic_store(to_atomic(a), v, to_mo(mo)); } -#if __TSAN_HAS_INT128 && !defined(SANITIZER_GO) +#if __TSAN_HAS_INT128 && !SANITIZER_GO static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) { SpinMutexLock lock(&mutex128); *a = v; @@ -455,7 +432,7 @@ static T AtomicCAS(ThreadState *thr, uptr pc, return c; } -#ifndef SANITIZER_GO +#if !SANITIZER_GO static void NoTsanAtomicFence(morder mo) { __sync_synchronize(); } @@ -467,7 +444,7 @@ static void AtomicFence(ThreadState *thr, uptr pc, morder mo) { #endif // Interface functions follow. -#ifndef SANITIZER_GO +#if !SANITIZER_GO // C/C++ @@ -866,7 +843,7 @@ void __tsan_atomic_signal_fence(morder mo) { } } // extern "C" -#else // #ifndef SANITIZER_GO +#else // #if !SANITIZER_GO // Go @@ -949,4 +926,4 @@ void __tsan_go_atomic64_compare_exchange( *(bool*)(a+24) = (cur == cmp); } } // extern "C" -#endif // #ifndef SANITIZER_GO +#endif // #if !SANITIZER_GO diff --git a/libsanitizer/tsan/tsan_interface_java.cc b/libsanitizer/tsan/tsan_interface_java.cc index 934ccc3ef9c8..d1638f2aa9d3 100644 --- a/libsanitizer/tsan/tsan_interface_java.cc +++ b/libsanitizer/tsan/tsan_interface_java.cc @@ -109,7 +109,7 @@ void __tsan_java_free(jptr ptr, jptr size) { CHECK_GE(ptr, jctx->heap_begin); CHECK_LE(ptr + size, jctx->heap_begin + jctx->heap_size); - ctx->metamap.FreeRange(thr, pc, ptr, size); + ctx->metamap.FreeRange(thr->proc(), ptr, size); } void __tsan_java_move(jptr src, jptr dst, jptr size) { diff --git a/libsanitizer/tsan/tsan_libdispatch_mac.cc b/libsanitizer/tsan/tsan_libdispatch_mac.cc index 5b39665d5d20..10c70a831c63 100644 --- a/libsanitizer/tsan/tsan_libdispatch_mac.cc +++ b/libsanitizer/tsan/tsan_libdispatch_mac.cc @@ -19,11 +19,205 @@ #include "tsan_platform.h" #include "tsan_rtl.h" +#include <Block.h> #include <dispatch/dispatch.h> #include <pthread.h> +typedef long long_t; // NOLINT + namespace __tsan { +typedef struct { + dispatch_queue_t queue; + void *orig_context; + dispatch_function_t orig_work; + bool free_context_in_callback; + bool submitted_synchronously; + bool is_barrier_block; + uptr non_queue_sync_object; +} tsan_block_context_t; + +// The offsets of different fields of the dispatch_queue_t structure, exported +// by libdispatch.dylib. +extern "C" struct dispatch_queue_offsets_s { + const uint16_t dqo_version; + const uint16_t dqo_label; + const uint16_t dqo_label_size; + const uint16_t dqo_flags; + const uint16_t dqo_flags_size; + const uint16_t dqo_serialnum; + const uint16_t dqo_serialnum_size; + const uint16_t dqo_width; + const uint16_t dqo_width_size; + const uint16_t dqo_running; + const uint16_t dqo_running_size; + const uint16_t dqo_suspend_cnt; + const uint16_t dqo_suspend_cnt_size; + const uint16_t dqo_target_queue; + const uint16_t dqo_target_queue_size; + const uint16_t dqo_priority; + const uint16_t dqo_priority_size; +} dispatch_queue_offsets; + +static bool IsQueueSerial(dispatch_queue_t q) { + CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2); + uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width); + CHECK_NE(width, 0); + return width == 1; +} + +static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) { + CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8); + dispatch_queue_t target_queue = + *(dispatch_queue_t *)(((uptr)source) + + dispatch_queue_offsets.dqo_target_queue); + CHECK_NE(target_queue, 0); + return target_queue; +} + +static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc, + dispatch_queue_t queue, + void *orig_context, + dispatch_function_t orig_work) { + tsan_block_context_t *new_context = + (tsan_block_context_t *)user_alloc(thr, pc, sizeof(tsan_block_context_t)); + new_context->queue = queue; + new_context->orig_context = orig_context; + new_context->orig_work = orig_work; + new_context->free_context_in_callback = true; + new_context->submitted_synchronously = false; + new_context->is_barrier_block = false; + return new_context; +} + +static void dispatch_callback_wrap(void *param) { + SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap); + tsan_block_context_t *context = (tsan_block_context_t *)param; + bool is_queue_serial = context->queue && IsQueueSerial(context->queue); + uptr sync_ptr = (uptr)context->queue ?: context->non_queue_sync_object; + + uptr serial_sync = (uptr)sync_ptr; + uptr concurrent_sync = ((uptr)sync_ptr) + sizeof(uptr); + uptr submit_sync = (uptr)context; + bool serial_task = context->is_barrier_block || is_queue_serial; + + Acquire(thr, pc, submit_sync); + Acquire(thr, pc, serial_sync); + if (serial_task) Acquire(thr, pc, concurrent_sync); + + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + context->orig_work(context->orig_context); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + + Release(thr, pc, serial_task ? serial_sync : concurrent_sync); + if (context->submitted_synchronously) Release(thr, pc, submit_sync); + + if (context->free_context_in_callback) user_free(thr, pc, context); +} + +static void invoke_block(void *param) { + dispatch_block_t block = (dispatch_block_t)param; + block(); +} + +static void invoke_and_release_block(void *param) { + dispatch_block_t block = (dispatch_block_t)param; + block(); + Block_release(block); +} + +#define DISPATCH_INTERCEPT_B(name, barrier) \ + TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \ + SCOPED_TSAN_INTERCEPTOR(name, q, block); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ + dispatch_block_t heap_block = Block_copy(block); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ + tsan_block_context_t *new_context = \ + AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \ + new_context->is_barrier_block = barrier; \ + Release(thr, pc, (uptr)new_context); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ + REAL(name##_f)(q, new_context, dispatch_callback_wrap); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ + } + +#define DISPATCH_INTERCEPT_SYNC_B(name, barrier) \ + TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \ + SCOPED_TSAN_INTERCEPTOR(name, q, block); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ + dispatch_block_t heap_block = Block_copy(block); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ + tsan_block_context_t new_context = { \ + q, heap_block, &invoke_and_release_block, false, true, barrier, 0}; \ + Release(thr, pc, (uptr)&new_context); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ + REAL(name##_f)(q, &new_context, dispatch_callback_wrap); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ + Acquire(thr, pc, (uptr)&new_context); \ + } + +#define DISPATCH_INTERCEPT_F(name, barrier) \ + TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ + dispatch_function_t work) { \ + SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ + tsan_block_context_t *new_context = \ + AllocContext(thr, pc, q, context, work); \ + new_context->is_barrier_block = barrier; \ + Release(thr, pc, (uptr)new_context); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ + REAL(name)(q, new_context, dispatch_callback_wrap); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ + } + +#define DISPATCH_INTERCEPT_SYNC_F(name, barrier) \ + TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ + dispatch_function_t work) { \ + SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ + tsan_block_context_t new_context = { \ + q, context, work, false, true, barrier, 0}; \ + Release(thr, pc, (uptr)&new_context); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ + REAL(name)(q, &new_context, dispatch_callback_wrap); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ + Acquire(thr, pc, (uptr)&new_context); \ + } + +// We wrap dispatch_async, dispatch_sync and friends where we allocate a new +// context, which is used to synchronize (we release the context before +// submitting, and the callback acquires it before executing the original +// callback). +DISPATCH_INTERCEPT_B(dispatch_async, false) +DISPATCH_INTERCEPT_B(dispatch_barrier_async, true) +DISPATCH_INTERCEPT_F(dispatch_async_f, false) +DISPATCH_INTERCEPT_F(dispatch_barrier_async_f, true) +DISPATCH_INTERCEPT_SYNC_B(dispatch_sync, false) +DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_sync, true) +DISPATCH_INTERCEPT_SYNC_F(dispatch_sync_f, false) +DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_sync_f, true) + +TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when, + dispatch_queue_t queue, dispatch_block_t block) { + SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + dispatch_block_t heap_block = Block_copy(block); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + tsan_block_context_t *new_context = + AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block); + Release(thr, pc, (uptr)new_context); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); +} + +TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when, + dispatch_queue_t queue, void *context, + dispatch_function_t work) { + SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work); + WRAP(dispatch_after)(when, queue, ^(void) { + work(context); + }); +} + // GCD's dispatch_once implementation has a fast path that contains a racy read // and it's inlined into user's code. Furthermore, this fast path doesn't // establish a proper happens-before relations between the initialization and @@ -39,12 +233,14 @@ namespace __tsan { #undef dispatch_once TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate, dispatch_block_t block) { - SCOPED_TSAN_INTERCEPTOR(dispatch_once, predicate, block); + SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block); atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate); u32 v = atomic_load(a, memory_order_acquire); if (v == 0 && atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) { + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); block(); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); Release(thr, pc, (uptr)a); atomic_store(a, 2, memory_order_release); } else { @@ -59,10 +255,423 @@ TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate, #undef dispatch_once_f TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate, void *context, dispatch_function_t function) { - SCOPED_TSAN_INTERCEPTOR(dispatch_once_f, predicate, context, function); + SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); WRAP(dispatch_once)(predicate, ^(void) { function(context); }); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); +} + +TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal, + dispatch_semaphore_t dsema) { + SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema); + Release(thr, pc, (uptr)dsema); + return REAL(dispatch_semaphore_signal)(dsema); +} + +TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema, + dispatch_time_t timeout) { + SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout); + long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout); + if (result == 0) Acquire(thr, pc, (uptr)dsema); + return result; +} + +TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group, + dispatch_time_t timeout) { + SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout); + long_t result = REAL(dispatch_group_wait)(group, timeout); + if (result == 0) Acquire(thr, pc, (uptr)group); + return result; +} + +TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) { + SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group); + // Acquired in the group noticifaction callback in dispatch_group_notify[_f]. + Release(thr, pc, (uptr)group); + REAL(dispatch_group_leave)(group); +} + +TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group, + dispatch_queue_t queue, dispatch_block_t block) { + SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block); + dispatch_retain(group); + dispatch_group_enter(group); + __block dispatch_block_t block_copy = (dispatch_block_t)_Block_copy(block); + WRAP(dispatch_async)(queue, ^(void) { + block_copy(); + _Block_release(block_copy); + WRAP(dispatch_group_leave)(group); + dispatch_release(group); + }); +} + +TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group, + dispatch_queue_t queue, void *context, + dispatch_function_t work) { + SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work); + dispatch_retain(group); + dispatch_group_enter(group); + WRAP(dispatch_async)(queue, ^(void) { + work(context); + WRAP(dispatch_group_leave)(group); + dispatch_release(group); + }); +} + +TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group, + dispatch_queue_t q, dispatch_block_t block) { + SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block); + + // To make sure the group is still available in the callback (otherwise + // it can be already destroyed). Will be released in the callback. + dispatch_retain(group); + + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + dispatch_block_t heap_block = Block_copy(^(void) { + { + SCOPED_INTERCEPTOR_RAW(dispatch_read_callback); + // Released when leaving the group (dispatch_group_leave). + Acquire(thr, pc, (uptr)group); + } + dispatch_release(group); + block(); + }); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + tsan_block_context_t *new_context = + AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); + new_context->is_barrier_block = true; + Release(thr, pc, (uptr)new_context); + REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap); +} + +TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group, + dispatch_queue_t q, void *context, dispatch_function_t work) { + WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); }); +} + +TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler, + dispatch_source_t source, dispatch_block_t handler) { + SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler); + if (handler == nullptr) + return REAL(dispatch_source_set_event_handler)(source, nullptr); + dispatch_queue_t q = GetTargetQueueFromSource(source); + __block tsan_block_context_t new_context = { + q, handler, &invoke_block, false, false, false, 0 }; + dispatch_block_t new_handler = Block_copy(^(void) { + new_context.orig_context = handler; // To explicitly capture "handler". + dispatch_callback_wrap(&new_context); + }); + uptr submit_sync = (uptr)&new_context; + Release(thr, pc, submit_sync); + REAL(dispatch_source_set_event_handler)(source, new_handler); + Block_release(new_handler); +} + +TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f, + dispatch_source_t source, dispatch_function_t handler) { + SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler); + if (handler == nullptr) + return REAL(dispatch_source_set_event_handler)(source, nullptr); + dispatch_block_t block = ^(void) { + handler(dispatch_get_context(source)); + }; + WRAP(dispatch_source_set_event_handler)(source, block); +} + +TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler, + dispatch_source_t source, dispatch_block_t handler) { + SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler); + if (handler == nullptr) + return REAL(dispatch_source_set_cancel_handler)(source, nullptr); + dispatch_queue_t q = GetTargetQueueFromSource(source); + __block tsan_block_context_t new_context = { + q, handler, &invoke_block, false, false, false, 0}; + dispatch_block_t new_handler = Block_copy(^(void) { + new_context.orig_context = handler; // To explicitly capture "handler". + dispatch_callback_wrap(&new_context); + }); + uptr submit_sync = (uptr)&new_context; + Release(thr, pc, submit_sync); + REAL(dispatch_source_set_cancel_handler)(source, new_handler); + Block_release(new_handler); +} + +TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f, + dispatch_source_t source, dispatch_function_t handler) { + SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source, + handler); + if (handler == nullptr) + return REAL(dispatch_source_set_cancel_handler)(source, nullptr); + dispatch_block_t block = ^(void) { + handler(dispatch_get_context(source)); + }; + WRAP(dispatch_source_set_cancel_handler)(source, block); +} + +TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler, + dispatch_source_t source, dispatch_block_t handler) { + SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source, + handler); + if (handler == nullptr) + return REAL(dispatch_source_set_registration_handler)(source, nullptr); + dispatch_queue_t q = GetTargetQueueFromSource(source); + __block tsan_block_context_t new_context = { + q, handler, &invoke_block, false, false, false, 0}; + dispatch_block_t new_handler = Block_copy(^(void) { + new_context.orig_context = handler; // To explicitly capture "handler". + dispatch_callback_wrap(&new_context); + }); + uptr submit_sync = (uptr)&new_context; + Release(thr, pc, submit_sync); + REAL(dispatch_source_set_registration_handler)(source, new_handler); + Block_release(new_handler); +} + +TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f, + dispatch_source_t source, dispatch_function_t handler) { + SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source, + handler); + if (handler == nullptr) + return REAL(dispatch_source_set_registration_handler)(source, nullptr); + dispatch_block_t block = ^(void) { + handler(dispatch_get_context(source)); + }; + WRAP(dispatch_source_set_registration_handler)(source, block); +} + +TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations, + dispatch_queue_t queue, void (^block)(size_t)) { + SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block); + + void *parent_to_child_sync = nullptr; + uptr parent_to_child_sync_uptr = (uptr)&parent_to_child_sync; + void *child_to_parent_sync = nullptr; + uptr child_to_parent_sync_uptr = (uptr)&child_to_parent_sync; + + Release(thr, pc, parent_to_child_sync_uptr); + void (^new_block)(size_t) = ^(size_t iteration) { + SCOPED_INTERCEPTOR_RAW(dispatch_apply); + Acquire(thr, pc, parent_to_child_sync_uptr); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + block(iteration); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + Release(thr, pc, child_to_parent_sync_uptr); + }; + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + REAL(dispatch_apply)(iterations, queue, new_block); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + Acquire(thr, pc, child_to_parent_sync_uptr); +} + +TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations, + dispatch_queue_t queue, void *context, + void (*work)(void *, size_t)) { + SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work); + void (^new_block)(size_t) = ^(size_t iteration) { + work(context, iteration); + }; + WRAP(dispatch_apply)(iterations, queue, new_block); +} + +DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) +DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, long_t sz) + +TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer, + size_t size, dispatch_queue_t q, dispatch_block_t destructor) { + SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor); + if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT)) + return REAL(dispatch_data_create)(buffer, size, q, destructor); + + if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE) + destructor = ^(void) { WRAP(free)((void *)buffer); }; + else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP) + destructor = ^(void) { WRAP(munmap)((void *)buffer, size); }; + + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + dispatch_block_t heap_block = Block_copy(destructor); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + tsan_block_context_t *new_context = + AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); + uptr submit_sync = (uptr)new_context; + Release(thr, pc, submit_sync); + return REAL(dispatch_data_create)(buffer, size, q, ^(void) { + dispatch_callback_wrap(new_context); + }); +} + +typedef void (^fd_handler_t)(dispatch_data_t data, int error); +typedef void (^cleanup_handler_t)(int error); + +TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length, + dispatch_queue_t q, fd_handler_t h) { + SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h); + __block tsan_block_context_t new_context = { + q, nullptr, &invoke_block, false, false, false, 0}; + fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { + new_context.orig_context = ^(void) { + h(data, error); + }; + dispatch_callback_wrap(&new_context); + }); + uptr submit_sync = (uptr)&new_context; + Release(thr, pc, submit_sync); + REAL(dispatch_read)(fd, length, q, new_h); + Block_release(new_h); +} + +TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data, + dispatch_queue_t q, fd_handler_t h) { + SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h); + __block tsan_block_context_t new_context = { + q, nullptr, &invoke_block, false, false, false, 0}; + fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { + new_context.orig_context = ^(void) { + h(data, error); + }; + dispatch_callback_wrap(&new_context); + }); + uptr submit_sync = (uptr)&new_context; + Release(thr, pc, submit_sync); + REAL(dispatch_write)(fd, data, q, new_h); + Block_release(new_h); +} + +TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset, + size_t length, dispatch_queue_t q, dispatch_io_handler_t h) { + SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h); + __block tsan_block_context_t new_context = { + q, nullptr, &invoke_block, false, false, false, 0}; + dispatch_io_handler_t new_h = + Block_copy(^(bool done, dispatch_data_t data, int error) { + new_context.orig_context = ^(void) { + h(done, data, error); + }; + dispatch_callback_wrap(&new_context); + }); + uptr submit_sync = (uptr)&new_context; + Release(thr, pc, submit_sync); + REAL(dispatch_io_read)(channel, offset, length, q, new_h); + Block_release(new_h); +} + +TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset, + dispatch_data_t data, dispatch_queue_t q, + dispatch_io_handler_t h) { + SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h); + __block tsan_block_context_t new_context = { + q, nullptr, &invoke_block, false, false, false, 0}; + dispatch_io_handler_t new_h = + Block_copy(^(bool done, dispatch_data_t data, int error) { + new_context.orig_context = ^(void) { + h(done, data, error); + }; + dispatch_callback_wrap(&new_context); + }); + uptr submit_sync = (uptr)&new_context; + Release(thr, pc, submit_sync); + REAL(dispatch_io_write)(channel, offset, data, q, new_h); + Block_release(new_h); +} + +TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel, + dispatch_block_t barrier) { + SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier); + __block tsan_block_context_t new_context = { + nullptr, nullptr, &invoke_block, false, false, false, 0}; + new_context.non_queue_sync_object = (uptr)channel; + new_context.is_barrier_block = true; + dispatch_block_t new_block = Block_copy(^(void) { + new_context.orig_context = ^(void) { + barrier(); + }; + dispatch_callback_wrap(&new_context); + }); + uptr submit_sync = (uptr)&new_context; + Release(thr, pc, submit_sync); + REAL(dispatch_io_barrier)(channel, new_block); + Block_release(new_block); +} + +TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type, + dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) { + SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h); + __block dispatch_io_t new_channel = nullptr; + __block tsan_block_context_t new_context = { + q, nullptr, &invoke_block, false, false, false, 0}; + cleanup_handler_t new_h = Block_copy(^(int error) { + { + SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); + Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. + } + new_context.orig_context = ^(void) { + h(error); + }; + dispatch_callback_wrap(&new_context); + }); + uptr submit_sync = (uptr)&new_context; + Release(thr, pc, submit_sync); + new_channel = REAL(dispatch_io_create)(type, fd, q, new_h); + Block_release(new_h); + return new_channel; +} + +TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path, + dispatch_io_type_t type, const char *path, int oflag, + mode_t mode, dispatch_queue_t q, cleanup_handler_t h) { + SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode, + q, h); + __block dispatch_io_t new_channel = nullptr; + __block tsan_block_context_t new_context = { + q, nullptr, &invoke_block, false, false, false, 0}; + cleanup_handler_t new_h = Block_copy(^(int error) { + { + SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); + Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. + } + new_context.orig_context = ^(void) { + h(error); + }; + dispatch_callback_wrap(&new_context); + }); + uptr submit_sync = (uptr)&new_context; + Release(thr, pc, submit_sync); + new_channel = + REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h); + Block_release(new_h); + return new_channel; +} + +TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io, + dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q, + cleanup_handler_t h) { + SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h); + __block dispatch_io_t new_channel = nullptr; + __block tsan_block_context_t new_context = { + q, nullptr, &invoke_block, false, false, false, 0}; + cleanup_handler_t new_h = Block_copy(^(int error) { + { + SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); + Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. + } + new_context.orig_context = ^(void) { + h(error); + }; + dispatch_callback_wrap(&new_context); + }); + uptr submit_sync = (uptr)&new_context; + Release(thr, pc, submit_sync); + new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h); + Block_release(new_h); + return new_channel; +} + +TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel, + dispatch_io_close_flags_t flags) { + SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags); + Release(thr, pc, (uptr)channel); // Acquire() in dispatch_io_create[_*]. + return REAL(dispatch_io_close)(channel, flags); } } // namespace __tsan diff --git a/libsanitizer/tsan/tsan_malloc_mac.cc b/libsanitizer/tsan/tsan_malloc_mac.cc index 97090773ca76..cc1031a584fa 100644 --- a/libsanitizer/tsan/tsan_malloc_mac.cc +++ b/libsanitizer/tsan/tsan_malloc_mac.cc @@ -25,41 +25,32 @@ using namespace __tsan; #define COMMON_MALLOC_MEMALIGN(alignment, size) \ void *p = \ user_alloc(cur_thread(), StackTrace::GetCurrentPc(), size, alignment) -#define COMMON_MALLOC_MALLOC(size) \ - if (cur_thread()->in_symbolizer) \ - return REAL(malloc)(size); \ - SCOPED_INTERCEPTOR_RAW(malloc, size); \ +#define COMMON_MALLOC_MALLOC(size) \ + if (cur_thread()->in_symbolizer) return InternalAlloc(size); \ + SCOPED_INTERCEPTOR_RAW(malloc, size); \ void *p = user_alloc(thr, pc, size) -#define COMMON_MALLOC_REALLOC(ptr, size) \ - if (cur_thread()->in_symbolizer) \ - return REAL(realloc)(ptr, size); \ - SCOPED_INTERCEPTOR_RAW(realloc, ptr, size); \ +#define COMMON_MALLOC_REALLOC(ptr, size) \ + if (cur_thread()->in_symbolizer) return InternalRealloc(ptr, size); \ + SCOPED_INTERCEPTOR_RAW(realloc, ptr, size); \ void *p = user_realloc(thr, pc, ptr, size) -#define COMMON_MALLOC_CALLOC(count, size) \ - if (cur_thread()->in_symbolizer) \ - return REAL(calloc)(count, size); \ - SCOPED_INTERCEPTOR_RAW(calloc, size, count); \ +#define COMMON_MALLOC_CALLOC(count, size) \ + if (cur_thread()->in_symbolizer) return InternalCalloc(count, size); \ + SCOPED_INTERCEPTOR_RAW(calloc, size, count); \ void *p = user_calloc(thr, pc, size, count) -#define COMMON_MALLOC_VALLOC(size) \ - if (cur_thread()->in_symbolizer) \ - return REAL(valloc)(size); \ - SCOPED_INTERCEPTOR_RAW(valloc, size); \ +#define COMMON_MALLOC_VALLOC(size) \ + if (cur_thread()->in_symbolizer) \ + return InternalAlloc(size, nullptr, GetPageSizeCached()); \ + SCOPED_INTERCEPTOR_RAW(valloc, size); \ void *p = user_alloc(thr, pc, size, GetPageSizeCached()) -#define COMMON_MALLOC_FREE(ptr) \ - if (cur_thread()->in_symbolizer) \ - return REAL(free)(ptr); \ - SCOPED_INTERCEPTOR_RAW(free, ptr); \ +#define COMMON_MALLOC_FREE(ptr) \ + if (cur_thread()->in_symbolizer) return InternalFree(ptr); \ + SCOPED_INTERCEPTOR_RAW(free, ptr); \ user_free(thr, pc, ptr) -#define COMMON_MALLOC_SIZE(ptr) \ - uptr size = user_alloc_usable_size(ptr); +#define COMMON_MALLOC_SIZE(ptr) uptr size = user_alloc_usable_size(ptr); #define COMMON_MALLOC_FILL_STATS(zone, stats) #define COMMON_MALLOC_REPORT_UNKNOWN_REALLOC(ptr, zone_ptr, zone_name) \ (void)zone_name; \ Report("mz_realloc(%p) -- attempting to realloc unallocated memory.\n", ptr); -#define COMMON_MALLOC_IGNORE_INVALID_FREE false -#define COMMON_MALLOC_REPORT_FREE_UNALLOCATED(ptr, zone_ptr, zone_name) \ - (void)zone_name; \ - Report("free_common(%p) -- attempting to free unallocated memory.\n", ptr); #define COMMON_MALLOC_NAMESPACE __tsan #include "sanitizer_common/sanitizer_malloc_mac.inc" diff --git a/libsanitizer/tsan/tsan_mman.cc b/libsanitizer/tsan/tsan_mman.cc index c8a5a6a264a8..152c2de28d86 100644 --- a/libsanitizer/tsan/tsan_mman.cc +++ b/libsanitizer/tsan/tsan_mman.cc @@ -17,12 +17,14 @@ #include "tsan_flags.h" // May be overriden by front-end. -extern "C" void WEAK __sanitizer_malloc_hook(void *ptr, uptr size) { +SANITIZER_WEAK_DEFAULT_IMPL +void __sanitizer_malloc_hook(void *ptr, uptr size) { (void)ptr; (void)size; } -extern "C" void WEAK __sanitizer_free_hook(void *ptr) { +SANITIZER_WEAK_DEFAULT_IMPL +void __sanitizer_free_hook(void *ptr) { (void)ptr; } @@ -50,7 +52,7 @@ struct MapUnmapCallback { diff = p + size - RoundDown(p + size, kPageSize); if (diff != 0) size -= diff; - FlushUnneededShadowMemory((uptr)MemToMeta(p), size / kMetaRatio); + ReleaseMemoryToOS((uptr)MemToMeta(p), size / kMetaRatio); } }; @@ -59,18 +61,69 @@ Allocator *allocator() { return reinterpret_cast<Allocator*>(&allocator_placeholder); } +struct GlobalProc { + Mutex mtx; + Processor *proc; + + GlobalProc() + : mtx(MutexTypeGlobalProc, StatMtxGlobalProc) + , proc(ProcCreate()) { + } +}; + +static char global_proc_placeholder[sizeof(GlobalProc)] ALIGNED(64); +GlobalProc *global_proc() { + return reinterpret_cast<GlobalProc*>(&global_proc_placeholder); +} + +ScopedGlobalProcessor::ScopedGlobalProcessor() { + GlobalProc *gp = global_proc(); + ThreadState *thr = cur_thread(); + if (thr->proc()) + return; + // If we don't have a proc, use the global one. + // There are currently only two known case where this path is triggered: + // __interceptor_free + // __nptl_deallocate_tsd + // start_thread + // clone + // and: + // ResetRange + // __interceptor_munmap + // __deallocate_stack + // start_thread + // clone + // Ideally, we destroy thread state (and unwire proc) when a thread actually + // exits (i.e. when we join/wait it). Then we would not need the global proc + gp->mtx.Lock(); + ProcWire(gp->proc, thr); +} + +ScopedGlobalProcessor::~ScopedGlobalProcessor() { + GlobalProc *gp = global_proc(); + ThreadState *thr = cur_thread(); + if (thr->proc() != gp->proc) + return; + ProcUnwire(gp->proc, thr); + gp->mtx.Unlock(); +} + void InitializeAllocator() { allocator()->Init(common_flags()->allocator_may_return_null); } -void AllocatorThreadStart(ThreadState *thr) { - allocator()->InitCache(&thr->alloc_cache); - internal_allocator()->InitCache(&thr->internal_alloc_cache); +void InitializeAllocatorLate() { + new(global_proc()) GlobalProc(); +} + +void AllocatorProcStart(Processor *proc) { + allocator()->InitCache(&proc->alloc_cache); + internal_allocator()->InitCache(&proc->internal_alloc_cache); } -void AllocatorThreadFinish(ThreadState *thr) { - allocator()->DestroyCache(&thr->alloc_cache); - internal_allocator()->DestroyCache(&thr->internal_alloc_cache); +void AllocatorProcFinish(Processor *proc) { + allocator()->DestroyCache(&proc->alloc_cache); + internal_allocator()->DestroyCache(&proc->internal_alloc_cache); } void AllocatorPrintStats() { @@ -93,8 +146,8 @@ static void SignalUnsafeCall(ThreadState *thr, uptr pc) { void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align, bool signal) { if ((sz >= (1ull << 40)) || (align >= (1ull << 40))) - return allocator()->ReturnNullOrDie(); - void *p = allocator()->Allocate(&thr->alloc_cache, sz, align); + return allocator()->ReturnNullOrDieOnBadRequest(); + void *p = allocator()->Allocate(&thr->proc()->alloc_cache, sz, align); if (p == 0) return 0; if (ctx && ctx->initialized) @@ -106,7 +159,7 @@ void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align, bool signal) { void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) { if (CallocShouldReturnNullDueToOverflow(size, n)) - return allocator()->ReturnNullOrDie(); + return allocator()->ReturnNullOrDieOnBadRequest(); void *p = user_alloc(thr, pc, n * size); if (p) internal_memset(p, 0, n * size); @@ -114,9 +167,10 @@ void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) { } void user_free(ThreadState *thr, uptr pc, void *p, bool signal) { + ScopedGlobalProcessor sgp; if (ctx && ctx->initialized) OnUserFree(thr, pc, (uptr)p, true); - allocator()->Deallocate(&thr->alloc_cache, p); + allocator()->Deallocate(&thr->proc()->alloc_cache, p); if (signal) SignalUnsafeCall(thr, pc); } @@ -132,27 +186,23 @@ void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) { void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write) { CHECK_NE(p, (void*)0); - uptr sz = ctx->metamap.FreeBlock(thr, pc, p); + uptr sz = ctx->metamap.FreeBlock(thr->proc(), p); DPrintf("#%d: free(%p, %zu)\n", thr->tid, p, sz); if (write && thr->ignore_reads_and_writes == 0) MemoryRangeFreed(thr, pc, (uptr)p, sz); } void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz) { - void *p2 = 0; // FIXME: Handle "shrinking" more efficiently, // it seems that some software actually does this. - if (sz) { - p2 = user_alloc(thr, pc, sz); - if (p2 == 0) - return 0; - if (p) { - uptr oldsz = user_alloc_usable_size(p); - internal_memcpy(p2, p, min(oldsz, sz)); - } - } - if (p) + void *p2 = user_alloc(thr, pc, sz); + if (p2 == 0) + return 0; + if (p) { + uptr oldsz = user_alloc_usable_size(p); + internal_memcpy(p2, p, min(oldsz, sz)); user_free(thr, pc, p); + } return p2; } @@ -160,7 +210,11 @@ uptr user_alloc_usable_size(const void *p) { if (p == 0) return 0; MBlock *b = ctx->metamap.GetBlock((uptr)p); - return b ? b->siz : 0; + if (!b) + return 0; // Not a valid pointer. + if (b->siz == 0) + return 1; // Zero-sized allocations are actually 1 byte. + return b->siz; } void invoke_malloc_hook(void *ptr, uptr size) { @@ -168,6 +222,7 @@ void invoke_malloc_hook(void *ptr, uptr size) { if (ctx == 0 || !ctx->initialized || thr->ignore_interceptors) return; __sanitizer_malloc_hook(ptr, size); + RunMallocHooks(ptr, size); } void invoke_free_hook(void *ptr) { @@ -175,6 +230,7 @@ void invoke_free_hook(void *ptr) { if (ctx == 0 || !ctx->initialized || thr->ignore_interceptors) return; __sanitizer_free_hook(ptr); + RunFreeHooks(ptr); } void *internal_alloc(MBlockType typ, uptr sz) { @@ -183,7 +239,7 @@ void *internal_alloc(MBlockType typ, uptr sz) { thr->nomalloc = 0; // CHECK calls internal_malloc(). CHECK(0); } - return InternalAlloc(sz, &thr->internal_alloc_cache); + return InternalAlloc(sz, &thr->proc()->internal_alloc_cache); } void internal_free(void *p) { @@ -192,7 +248,7 @@ void internal_free(void *p) { thr->nomalloc = 0; // CHECK calls internal_malloc(). CHECK(0); } - InternalFree(p, &thr->internal_alloc_cache); + InternalFree(p, &thr->proc()->internal_alloc_cache); } } // namespace __tsan @@ -234,8 +290,8 @@ uptr __sanitizer_get_allocated_size(const void *p) { void __tsan_on_thread_idle() { ThreadState *thr = cur_thread(); - allocator()->SwallowCache(&thr->alloc_cache); - internal_allocator()->SwallowCache(&thr->internal_alloc_cache); - ctx->metamap.OnThreadIdle(thr); + allocator()->SwallowCache(&thr->proc()->alloc_cache); + internal_allocator()->SwallowCache(&thr->proc()->internal_alloc_cache); + ctx->metamap.OnProcIdle(thr->proc()); } } // extern "C" diff --git a/libsanitizer/tsan/tsan_mman.h b/libsanitizer/tsan/tsan_mman.h index a7c91fae5e16..d49c9a9e9f24 100644 --- a/libsanitizer/tsan/tsan_mman.h +++ b/libsanitizer/tsan/tsan_mman.h @@ -18,9 +18,10 @@ namespace __tsan { const uptr kDefaultAlignment = 16; void InitializeAllocator(); +void InitializeAllocatorLate(); void ReplaceSystemMalloc(); -void AllocatorThreadStart(ThreadState *thr); -void AllocatorThreadFinish(ThreadState *thr); +void AllocatorProcStart(Processor *proc); +void AllocatorProcFinish(Processor *proc); void AllocatorPrintStats(); // For user allocations. diff --git a/libsanitizer/tsan/tsan_mutex.cc b/libsanitizer/tsan/tsan_mutex.cc index 3d3f6cc82b9a..9b105cf201be 100644 --- a/libsanitizer/tsan/tsan_mutex.cc +++ b/libsanitizer/tsan/tsan_mutex.cc @@ -41,6 +41,7 @@ static MutexType CanLockTab[MutexTypeCount][MutexTypeCount] = { /*11 MutexTypeDDetector*/ {}, /*12 MutexTypeFired*/ {MutexTypeLeaf}, /*13 MutexTypeRacy*/ {MutexTypeLeaf}, + /*14 MutexTypeGlobalProc*/ {}, }; static bool CanLockAdj[MutexTypeCount][MutexTypeCount]; diff --git a/libsanitizer/tsan/tsan_mutex.h b/libsanitizer/tsan/tsan_mutex.h index 9960eee9bc8d..bd1000f0b29e 100644 --- a/libsanitizer/tsan/tsan_mutex.h +++ b/libsanitizer/tsan/tsan_mutex.h @@ -32,6 +32,7 @@ enum MutexType { MutexTypeDDetector, MutexTypeFired, MutexTypeRacy, + MutexTypeGlobalProc, // This must be the last. MutexTypeCount diff --git a/libsanitizer/tsan/tsan_mutexset.h b/libsanitizer/tsan/tsan_mutexset.h index d5406ae9f128..b2c60b9a826f 100644 --- a/libsanitizer/tsan/tsan_mutexset.h +++ b/libsanitizer/tsan/tsan_mutexset.h @@ -41,7 +41,7 @@ class MutexSet { } private: -#ifndef SANITIZER_GO +#if !SANITIZER_GO uptr size_; Desc descs_[kMaxSize]; #endif @@ -53,7 +53,7 @@ class MutexSet { // Go does not have mutexes, so do not spend memory and time. // (Go sync.Mutex is actually a semaphore -- can be unlocked // in different goroutine). -#ifdef SANITIZER_GO +#if SANITIZER_GO MutexSet::MutexSet() {} void MutexSet::Add(u64 id, bool write, u64 epoch) {} void MutexSet::Del(u64 id, bool write) {} diff --git a/libsanitizer/tsan/tsan_new_delete.cc b/libsanitizer/tsan/tsan_new_delete.cc index ba8474e07141..606cdd659f40 100644 --- a/libsanitizer/tsan/tsan_new_delete.cc +++ b/libsanitizer/tsan/tsan_new_delete.cc @@ -21,14 +21,10 @@ struct nothrow_t {}; DECLARE_REAL(void *, malloc, uptr size) DECLARE_REAL(void, free, void *ptr) -#if SANITIZER_MAC -#define __libc_malloc REAL(malloc) -#define __libc_free REAL(free) -#endif #define OPERATOR_NEW_BODY(mangled_name) \ if (cur_thread()->in_symbolizer) \ - return __libc_malloc(size); \ + return InternalAlloc(size); \ void *p = 0; \ { \ SCOPED_INTERCEPTOR_RAW(mangled_name, size); \ @@ -64,7 +60,7 @@ void *operator new[](__sanitizer::uptr size, std::nothrow_t const&) { #define OPERATOR_DELETE_BODY(mangled_name) \ if (ptr == 0) return; \ if (cur_thread()->in_symbolizer) \ - return __libc_free(ptr); \ + return InternalFree(ptr); \ invoke_free_hook(ptr); \ SCOPED_INTERCEPTOR_RAW(mangled_name, ptr); \ user_free(thr, pc, ptr); diff --git a/libsanitizer/tsan/tsan_platform.h b/libsanitizer/tsan/tsan_platform.h index f34f577ec030..368edc21ce47 100644 --- a/libsanitizer/tsan/tsan_platform.h +++ b/libsanitizer/tsan/tsan_platform.h @@ -22,38 +22,46 @@ namespace __tsan { -#if !defined(SANITIZER_GO) +#if !SANITIZER_GO #if defined(__x86_64__) /* C/C++ on linux/x86_64 and freebsd/x86_64 -0000 0000 1000 - 0100 0000 0000: main binary and/or MAP_32BIT mappings -0100 0000 0000 - 0200 0000 0000: - -0200 0000 0000 - 1000 0000 0000: shadow -1000 0000 0000 - 3000 0000 0000: - +0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB) +0040 0000 0000 - 0100 0000 0000: - +0100 0000 0000 - 2000 0000 0000: shadow +2000 0000 0000 - 3000 0000 0000: - 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects) -4000 0000 0000 - 6000 0000 0000: - +4000 0000 0000 - 5500 0000 0000: - +5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels +5680 0000 0000 - 6000 0000 0000: - 6000 0000 0000 - 6200 0000 0000: traces 6200 0000 0000 - 7d00 0000 0000: - -7d00 0000 0000 - 7e00 0000 0000: heap -7e00 0000 0000 - 7e80 0000 0000: - +7b00 0000 0000 - 7c00 0000 0000: heap +7c00 0000 0000 - 7e80 0000 0000: - 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack */ -const uptr kMetaShadowBeg = 0x300000000000ull; -const uptr kMetaShadowEnd = 0x400000000000ull; -const uptr kTraceMemBeg = 0x600000000000ull; -const uptr kTraceMemEnd = 0x620000000000ull; -const uptr kShadowBeg = 0x020000000000ull; -const uptr kShadowEnd = 0x100000000000ull; -const uptr kHeapMemBeg = 0x7d0000000000ull; -const uptr kHeapMemEnd = 0x7e0000000000ull; -const uptr kLoAppMemBeg = 0x000000001000ull; -const uptr kLoAppMemEnd = 0x010000000000ull; -const uptr kHiAppMemBeg = 0x7e8000000000ull; -const uptr kHiAppMemEnd = 0x800000000000ull; -const uptr kAppMemMsk = 0x7c0000000000ull; -const uptr kAppMemXor = 0x020000000000ull; -const uptr kVdsoBeg = 0xf000000000000000ull; +struct Mapping { + static const uptr kMetaShadowBeg = 0x300000000000ull; + static const uptr kMetaShadowEnd = 0x340000000000ull; + static const uptr kTraceMemBeg = 0x600000000000ull; + static const uptr kTraceMemEnd = 0x620000000000ull; + static const uptr kShadowBeg = 0x010000000000ull; + static const uptr kShadowEnd = 0x200000000000ull; + static const uptr kHeapMemBeg = 0x7b0000000000ull; + static const uptr kHeapMemEnd = 0x7c0000000000ull; + static const uptr kLoAppMemBeg = 0x000000001000ull; + static const uptr kLoAppMemEnd = 0x008000000000ull; + static const uptr kMidAppMemBeg = 0x550000000000ull; + static const uptr kMidAppMemEnd = 0x568000000000ull; + static const uptr kHiAppMemBeg = 0x7e8000000000ull; + static const uptr kHiAppMemEnd = 0x800000000000ull; + static const uptr kAppMemMsk = 0x780000000000ull; + static const uptr kAppMemXor = 0x040000000000ull; + static const uptr kVdsoBeg = 0xf000000000000000ull; +}; + +#define TSAN_MID_APP_RANGE 1 #elif defined(__mips64) /* C/C++ on linux/mips64 @@ -69,129 +77,204 @@ fe00 0000 00 - ff00 0000 00: heap ff00 0000 00 - ff80 0000 00: - ff80 0000 00 - ffff ffff ff: modules and main thread stack */ -const uptr kMetaShadowBeg = 0x3000000000ull; -const uptr kMetaShadowEnd = 0x4000000000ull; -const uptr kTraceMemBeg = 0x6000000000ull; -const uptr kTraceMemEnd = 0x6200000000ull; -const uptr kShadowBeg = 0x1400000000ull; -const uptr kShadowEnd = 0x2400000000ull; -const uptr kHeapMemBeg = 0xfe00000000ull; -const uptr kHeapMemEnd = 0xff00000000ull; -const uptr kLoAppMemBeg = 0x0100000000ull; -const uptr kLoAppMemEnd = 0x0200000000ull; -const uptr kHiAppMemBeg = 0xff80000000ull; -const uptr kHiAppMemEnd = 0xffffffffffull; -const uptr kAppMemMsk = 0xfc00000000ull; -const uptr kAppMemXor = 0x0400000000ull; -const uptr kVdsoBeg = 0xfffff00000ull; +struct Mapping { + static const uptr kMetaShadowBeg = 0x4000000000ull; + static const uptr kMetaShadowEnd = 0x5000000000ull; + static const uptr kTraceMemBeg = 0xb000000000ull; + static const uptr kTraceMemEnd = 0xb200000000ull; + static const uptr kShadowBeg = 0x2400000000ull; + static const uptr kShadowEnd = 0x4000000000ull; + static const uptr kHeapMemBeg = 0xfe00000000ull; + static const uptr kHeapMemEnd = 0xff00000000ull; + static const uptr kLoAppMemBeg = 0x0100000000ull; + static const uptr kLoAppMemEnd = 0x0200000000ull; + static const uptr kMidAppMemBeg = 0xaa00000000ull; + static const uptr kMidAppMemEnd = 0xab00000000ull; + static const uptr kHiAppMemBeg = 0xff80000000ull; + static const uptr kHiAppMemEnd = 0xffffffffffull; + static const uptr kAppMemMsk = 0xf800000000ull; + static const uptr kAppMemXor = 0x0800000000ull; + static const uptr kVdsoBeg = 0xfffff00000ull; +}; + +#define TSAN_MID_APP_RANGE 1 #elif defined(__aarch64__) -# if SANITIZER_AARCH64_VMA == 39 +// AArch64 supports multiple VMA which leads to multiple address transformation +// functions. To support these multiple VMAS transformations and mappings TSAN +// runtime for AArch64 uses an external memory read (vmaSize) to select which +// mapping to use. Although slower, it make a same instrumented binary run on +// multiple kernels. + /* C/C++ on linux/aarch64 (39-bit VMA) -0000 4000 00 - 0200 0000 00: main binary -2000 0000 00 - 4000 0000 00: shadow memory -4000 0000 00 - 5000 0000 00: metainfo -5000 0000 00 - 6000 0000 00: - +0000 0010 00 - 0100 0000 00: main binary +0100 0000 00 - 0800 0000 00: - +0800 0000 00 - 2000 0000 00: shadow memory +2000 0000 00 - 3100 0000 00: - +3100 0000 00 - 3400 0000 00: metainfo +3400 0000 00 - 5500 0000 00: - +5500 0000 00 - 5600 0000 00: main binary (PIE) +5600 0000 00 - 6000 0000 00: - 6000 0000 00 - 6200 0000 00: traces 6200 0000 00 - 7d00 0000 00: - -7d00 0000 00 - 7e00 0000 00: heap -7e00 0000 00 - 7fff ffff ff: modules and main thread stack +7c00 0000 00 - 7d00 0000 00: heap +7d00 0000 00 - 7fff ffff ff: modules and main thread stack */ -const uptr kLoAppMemBeg = 0x0000400000ull; -const uptr kLoAppMemEnd = 0x0200000000ull; -const uptr kShadowBeg = 0x2000000000ull; -const uptr kShadowEnd = 0x4000000000ull; -const uptr kMetaShadowBeg = 0x4000000000ull; -const uptr kMetaShadowEnd = 0x5000000000ull; -const uptr kTraceMemBeg = 0x6000000000ull; -const uptr kTraceMemEnd = 0x6200000000ull; -const uptr kHeapMemBeg = 0x7d00000000ull; -const uptr kHeapMemEnd = 0x7e00000000ull; -const uptr kHiAppMemBeg = 0x7e00000000ull; -const uptr kHiAppMemEnd = 0x7fffffffffull; -const uptr kAppMemMsk = 0x7800000000ull; -const uptr kAppMemXor = 0x0800000000ull; -const uptr kVdsoBeg = 0x7f00000000ull; -# elif SANITIZER_AARCH64_VMA == 42 +struct Mapping39 { + static const uptr kLoAppMemBeg = 0x0000001000ull; + static const uptr kLoAppMemEnd = 0x0100000000ull; + static const uptr kShadowBeg = 0x0800000000ull; + static const uptr kShadowEnd = 0x2000000000ull; + static const uptr kMetaShadowBeg = 0x3100000000ull; + static const uptr kMetaShadowEnd = 0x3400000000ull; + static const uptr kMidAppMemBeg = 0x5500000000ull; + static const uptr kMidAppMemEnd = 0x5600000000ull; + static const uptr kTraceMemBeg = 0x6000000000ull; + static const uptr kTraceMemEnd = 0x6200000000ull; + static const uptr kHeapMemBeg = 0x7c00000000ull; + static const uptr kHeapMemEnd = 0x7d00000000ull; + static const uptr kHiAppMemBeg = 0x7e00000000ull; + static const uptr kHiAppMemEnd = 0x7fffffffffull; + static const uptr kAppMemMsk = 0x7800000000ull; + static const uptr kAppMemXor = 0x0200000000ull; + static const uptr kVdsoBeg = 0x7f00000000ull; +}; + /* C/C++ on linux/aarch64 (42-bit VMA) -00000 4000 00 - 01000 0000 00: main binary +00000 0010 00 - 01000 0000 00: main binary 01000 0000 00 - 10000 0000 00: - 10000 0000 00 - 20000 0000 00: shadow memory 20000 0000 00 - 26000 0000 00: - 26000 0000 00 - 28000 0000 00: metainfo -28000 0000 00 - 36200 0000 00: - +28000 0000 00 - 2aa00 0000 00: - +2aa00 0000 00 - 2ab00 0000 00: main binary (PIE) +2ab00 0000 00 - 36200 0000 00: - 36200 0000 00 - 36240 0000 00: traces 36240 0000 00 - 3e000 0000 00: - 3e000 0000 00 - 3f000 0000 00: heap -3c000 0000 00 - 3ff00 0000 00: - -3ff00 0000 00 - 3ffff f000 00: modules and main thread stack +3f000 0000 00 - 3ffff ffff ff: modules and main thread stack */ -const uptr kLoAppMemBeg = 0x00000400000ull; -const uptr kLoAppMemEnd = 0x01000000000ull; -const uptr kShadowBeg = 0x10000000000ull; -const uptr kShadowEnd = 0x20000000000ull; -const uptr kMetaShadowBeg = 0x26000000000ull; -const uptr kMetaShadowEnd = 0x28000000000ull; -const uptr kTraceMemBeg = 0x36200000000ull; -const uptr kTraceMemEnd = 0x36400000000ull; -const uptr kHeapMemBeg = 0x3e000000000ull; -const uptr kHeapMemEnd = 0x3f000000000ull; -const uptr kHiAppMemBeg = 0x3ff00000000ull; -const uptr kHiAppMemEnd = 0x3fffff00000ull; -const uptr kAppMemMsk = 0x3c000000000ull; -const uptr kAppMemXor = 0x04000000000ull; -const uptr kVdsoBeg = 0x37f00000000ull; -# endif -#endif - -ALWAYS_INLINE -bool IsAppMem(uptr mem) { - return (mem >= kHeapMemBeg && mem < kHeapMemEnd) || - (mem >= kLoAppMemBeg && mem < kLoAppMemEnd) || - (mem >= kHiAppMemBeg && mem < kHiAppMemEnd); -} - -ALWAYS_INLINE -bool IsShadowMem(uptr mem) { - return mem >= kShadowBeg && mem <= kShadowEnd; -} - -ALWAYS_INLINE -bool IsMetaMem(uptr mem) { - return mem >= kMetaShadowBeg && mem <= kMetaShadowEnd; -} +struct Mapping42 { + static const uptr kLoAppMemBeg = 0x00000001000ull; + static const uptr kLoAppMemEnd = 0x01000000000ull; + static const uptr kShadowBeg = 0x10000000000ull; + static const uptr kShadowEnd = 0x20000000000ull; + static const uptr kMetaShadowBeg = 0x26000000000ull; + static const uptr kMetaShadowEnd = 0x28000000000ull; + static const uptr kMidAppMemBeg = 0x2aa00000000ull; + static const uptr kMidAppMemEnd = 0x2ab00000000ull; + static const uptr kTraceMemBeg = 0x36200000000ull; + static const uptr kTraceMemEnd = 0x36400000000ull; + static const uptr kHeapMemBeg = 0x3e000000000ull; + static const uptr kHeapMemEnd = 0x3f000000000ull; + static const uptr kHiAppMemBeg = 0x3f000000000ull; + static const uptr kHiAppMemEnd = 0x3ffffffffffull; + static const uptr kAppMemMsk = 0x3c000000000ull; + static const uptr kAppMemXor = 0x04000000000ull; + static const uptr kVdsoBeg = 0x37f00000000ull; +}; -ALWAYS_INLINE -uptr MemToShadow(uptr x) { - DCHECK(IsAppMem(x)); - return (((x) & ~(kAppMemMsk | (kShadowCell - 1))) - ^ kAppMemXor) * kShadowCnt; -} +struct Mapping48 { + static const uptr kLoAppMemBeg = 0x0000000001000ull; + static const uptr kLoAppMemEnd = 0x0000200000000ull; + static const uptr kShadowBeg = 0x0002000000000ull; + static const uptr kShadowEnd = 0x0004000000000ull; + static const uptr kMetaShadowBeg = 0x0005000000000ull; + static const uptr kMetaShadowEnd = 0x0006000000000ull; + static const uptr kMidAppMemBeg = 0x0aaaa00000000ull; + static const uptr kMidAppMemEnd = 0x0aaaf00000000ull; + static const uptr kTraceMemBeg = 0x0f06000000000ull; + static const uptr kTraceMemEnd = 0x0f06200000000ull; + static const uptr kHeapMemBeg = 0x0ffff00000000ull; + static const uptr kHeapMemEnd = 0x0ffff00000000ull; + static const uptr kHiAppMemBeg = 0x0ffff00000000ull; + static const uptr kHiAppMemEnd = 0x1000000000000ull; + static const uptr kAppMemMsk = 0x0fff800000000ull; + static const uptr kAppMemXor = 0x0000800000000ull; + static const uptr kVdsoBeg = 0xffff000000000ull; +}; -ALWAYS_INLINE -u32 *MemToMeta(uptr x) { - DCHECK(IsAppMem(x)); - return (u32*)(((((x) & ~(kAppMemMsk | (kMetaShadowCell - 1))) - ^ kAppMemXor) / kMetaShadowCell * kMetaShadowSize) | kMetaShadowBeg); -} +// Indicates the runtime will define the memory regions at runtime. +#define TSAN_RUNTIME_VMA 1 +// Indicates that mapping defines a mid range memory segment. +#define TSAN_MID_APP_RANGE 1 +#elif defined(__powerpc64__) +// PPC64 supports multiple VMA which leads to multiple address transformation +// functions. To support these multiple VMAS transformations and mappings TSAN +// runtime for PPC64 uses an external memory read (vmaSize) to select which +// mapping to use. Although slower, it make a same instrumented binary run on +// multiple kernels. -ALWAYS_INLINE -uptr ShadowToMem(uptr s) { - CHECK(IsShadowMem(s)); - if (s >= MemToShadow(kLoAppMemBeg) && s <= MemToShadow(kLoAppMemEnd - 1)) - return (s / kShadowCnt) ^ kAppMemXor; - else - return ((s / kShadowCnt) ^ kAppMemXor) | kAppMemMsk; -} +/* +C/C++ on linux/powerpc64 (44-bit VMA) +0000 0000 0100 - 0001 0000 0000: main binary +0001 0000 0000 - 0001 0000 0000: - +0001 0000 0000 - 0b00 0000 0000: shadow +0b00 0000 0000 - 0b00 0000 0000: - +0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects) +0d00 0000 0000 - 0d00 0000 0000: - +0d00 0000 0000 - 0f00 0000 0000: traces +0f00 0000 0000 - 0f00 0000 0000: - +0f00 0000 0000 - 0f50 0000 0000: heap +0f50 0000 0000 - 0f60 0000 0000: - +0f60 0000 0000 - 1000 0000 0000: modules and main thread stack +*/ +struct Mapping44 { + static const uptr kMetaShadowBeg = 0x0b0000000000ull; + static const uptr kMetaShadowEnd = 0x0d0000000000ull; + static const uptr kTraceMemBeg = 0x0d0000000000ull; + static const uptr kTraceMemEnd = 0x0f0000000000ull; + static const uptr kShadowBeg = 0x000100000000ull; + static const uptr kShadowEnd = 0x0b0000000000ull; + static const uptr kLoAppMemBeg = 0x000000000100ull; + static const uptr kLoAppMemEnd = 0x000100000000ull; + static const uptr kHeapMemBeg = 0x0f0000000000ull; + static const uptr kHeapMemEnd = 0x0f5000000000ull; + static const uptr kHiAppMemBeg = 0x0f6000000000ull; + static const uptr kHiAppMemEnd = 0x100000000000ull; // 44 bits + static const uptr kAppMemMsk = 0x0f0000000000ull; + static const uptr kAppMemXor = 0x002100000000ull; + static const uptr kVdsoBeg = 0x3c0000000000000ull; +}; -static USED uptr UserRegions[] = { - kLoAppMemBeg, kLoAppMemEnd, - kHiAppMemBeg, kHiAppMemEnd, - kHeapMemBeg, kHeapMemEnd, +/* +C/C++ on linux/powerpc64 (46-bit VMA) +0000 0000 1000 - 0100 0000 0000: main binary +0100 0000 0000 - 0200 0000 0000: - +0100 0000 0000 - 1000 0000 0000: shadow +1000 0000 0000 - 1000 0000 0000: - +1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects) +2000 0000 0000 - 2000 0000 0000: - +2000 0000 0000 - 2200 0000 0000: traces +2200 0000 0000 - 3d00 0000 0000: - +3d00 0000 0000 - 3e00 0000 0000: heap +3e00 0000 0000 - 3e80 0000 0000: - +3e80 0000 0000 - 4000 0000 0000: modules and main thread stack +*/ +struct Mapping46 { + static const uptr kMetaShadowBeg = 0x100000000000ull; + static const uptr kMetaShadowEnd = 0x200000000000ull; + static const uptr kTraceMemBeg = 0x200000000000ull; + static const uptr kTraceMemEnd = 0x220000000000ull; + static const uptr kShadowBeg = 0x010000000000ull; + static const uptr kShadowEnd = 0x100000000000ull; + static const uptr kHeapMemBeg = 0x3d0000000000ull; + static const uptr kHeapMemEnd = 0x3e0000000000ull; + static const uptr kLoAppMemBeg = 0x000000001000ull; + static const uptr kLoAppMemEnd = 0x010000000000ull; + static const uptr kHiAppMemBeg = 0x3e8000000000ull; + static const uptr kHiAppMemEnd = 0x400000000000ull; // 46 bits + static const uptr kAppMemMsk = 0x3c0000000000ull; + static const uptr kAppMemXor = 0x020000000000ull; + static const uptr kVdsoBeg = 0x7800000000000000ull; }; -#elif defined(SANITIZER_GO) && !SANITIZER_WINDOWS +// Indicates the runtime will define the memory regions at runtime. +#define TSAN_RUNTIME_VMA 1 +#endif + +#elif SANITIZER_GO && !SANITIZER_WINDOWS /* Go on linux, darwin and freebsd 0000 0000 1000 - 0000 1000 0000: executable @@ -206,146 +289,529 @@ static USED uptr UserRegions[] = { 6200 0000 0000 - 8000 0000 0000: - */ -const uptr kMetaShadowBeg = 0x300000000000ull; -const uptr kMetaShadowEnd = 0x400000000000ull; -const uptr kTraceMemBeg = 0x600000000000ull; -const uptr kTraceMemEnd = 0x620000000000ull; -const uptr kShadowBeg = 0x200000000000ull; -const uptr kShadowEnd = 0x238000000000ull; -const uptr kAppMemBeg = 0x000000001000ull; -const uptr kAppMemEnd = 0x00e000000000ull; +struct Mapping { + static const uptr kMetaShadowBeg = 0x300000000000ull; + static const uptr kMetaShadowEnd = 0x400000000000ull; + static const uptr kTraceMemBeg = 0x600000000000ull; + static const uptr kTraceMemEnd = 0x620000000000ull; + static const uptr kShadowBeg = 0x200000000000ull; + static const uptr kShadowEnd = 0x238000000000ull; + static const uptr kAppMemBeg = 0x000000001000ull; + static const uptr kAppMemEnd = 0x00e000000000ull; +}; + +#elif SANITIZER_GO && SANITIZER_WINDOWS + +/* Go on windows +0000 0000 1000 - 0000 1000 0000: executable +0000 1000 0000 - 00f8 0000 0000: - +00c0 0000 0000 - 00e0 0000 0000: heap +00e0 0000 0000 - 0100 0000 0000: - +0100 0000 0000 - 0500 0000 0000: shadow +0500 0000 0000 - 0560 0000 0000: - +0560 0000 0000 - 0760 0000 0000: traces +0760 0000 0000 - 07d0 0000 0000: metainfo (memory blocks and sync objects) +07d0 0000 0000 - 8000 0000 0000: - +*/ + +struct Mapping { + static const uptr kMetaShadowBeg = 0x076000000000ull; + static const uptr kMetaShadowEnd = 0x07d000000000ull; + static const uptr kTraceMemBeg = 0x056000000000ull; + static const uptr kTraceMemEnd = 0x076000000000ull; + static const uptr kShadowBeg = 0x010000000000ull; + static const uptr kShadowEnd = 0x050000000000ull; + static const uptr kAppMemBeg = 0x000000001000ull; + static const uptr kAppMemEnd = 0x00e000000000ull; +}; + +#else +# error "Unknown platform" +#endif + + +#ifdef TSAN_RUNTIME_VMA +extern uptr vmaSize; +#endif + + +enum MappingType { + MAPPING_LO_APP_BEG, + MAPPING_LO_APP_END, + MAPPING_HI_APP_BEG, + MAPPING_HI_APP_END, +#ifdef TSAN_MID_APP_RANGE + MAPPING_MID_APP_BEG, + MAPPING_MID_APP_END, +#endif + MAPPING_HEAP_BEG, + MAPPING_HEAP_END, + MAPPING_APP_BEG, + MAPPING_APP_END, + MAPPING_SHADOW_BEG, + MAPPING_SHADOW_END, + MAPPING_META_SHADOW_BEG, + MAPPING_META_SHADOW_END, + MAPPING_TRACE_BEG, + MAPPING_TRACE_END, + MAPPING_VDSO_BEG, +}; + +template<typename Mapping, int Type> +uptr MappingImpl(void) { + switch (Type) { +#if !SANITIZER_GO + case MAPPING_LO_APP_BEG: return Mapping::kLoAppMemBeg; + case MAPPING_LO_APP_END: return Mapping::kLoAppMemEnd; +# ifdef TSAN_MID_APP_RANGE + case MAPPING_MID_APP_BEG: return Mapping::kMidAppMemBeg; + case MAPPING_MID_APP_END: return Mapping::kMidAppMemEnd; +# endif + case MAPPING_HI_APP_BEG: return Mapping::kHiAppMemBeg; + case MAPPING_HI_APP_END: return Mapping::kHiAppMemEnd; + case MAPPING_HEAP_BEG: return Mapping::kHeapMemBeg; + case MAPPING_HEAP_END: return Mapping::kHeapMemEnd; + case MAPPING_VDSO_BEG: return Mapping::kVdsoBeg; +#else + case MAPPING_APP_BEG: return Mapping::kAppMemBeg; + case MAPPING_APP_END: return Mapping::kAppMemEnd; +#endif + case MAPPING_SHADOW_BEG: return Mapping::kShadowBeg; + case MAPPING_SHADOW_END: return Mapping::kShadowEnd; + case MAPPING_META_SHADOW_BEG: return Mapping::kMetaShadowBeg; + case MAPPING_META_SHADOW_END: return Mapping::kMetaShadowEnd; + case MAPPING_TRACE_BEG: return Mapping::kTraceMemBeg; + case MAPPING_TRACE_END: return Mapping::kTraceMemEnd; + } +} +template<int Type> +uptr MappingArchImpl(void) { +#ifdef __aarch64__ + switch (vmaSize) { + case 39: return MappingImpl<Mapping39, Type>(); + case 42: return MappingImpl<Mapping42, Type>(); + case 48: return MappingImpl<Mapping48, Type>(); + } + DCHECK(0); + return 0; +#elif defined(__powerpc64__) + if (vmaSize == 44) + return MappingImpl<Mapping44, Type>(); + else + return MappingImpl<Mapping46, Type>(); + DCHECK(0); +#else + return MappingImpl<Mapping, Type>(); +#endif +} + +#if !SANITIZER_GO ALWAYS_INLINE -bool IsAppMem(uptr mem) { - return mem >= kAppMemBeg && mem < kAppMemEnd; +uptr LoAppMemBeg(void) { + return MappingArchImpl<MAPPING_LO_APP_BEG>(); +} +ALWAYS_INLINE +uptr LoAppMemEnd(void) { + return MappingArchImpl<MAPPING_LO_APP_END>(); } +#ifdef TSAN_MID_APP_RANGE ALWAYS_INLINE -bool IsShadowMem(uptr mem) { - return mem >= kShadowBeg && mem <= kShadowEnd; +uptr MidAppMemBeg(void) { + return MappingArchImpl<MAPPING_MID_APP_BEG>(); +} +ALWAYS_INLINE +uptr MidAppMemEnd(void) { + return MappingArchImpl<MAPPING_MID_APP_END>(); } +#endif ALWAYS_INLINE -bool IsMetaMem(uptr mem) { - return mem >= kMetaShadowBeg && mem <= kMetaShadowEnd; +uptr HeapMemBeg(void) { + return MappingArchImpl<MAPPING_HEAP_BEG>(); +} +ALWAYS_INLINE +uptr HeapMemEnd(void) { + return MappingArchImpl<MAPPING_HEAP_END>(); } ALWAYS_INLINE -uptr MemToShadow(uptr x) { - DCHECK(IsAppMem(x)); - return ((x & ~(kShadowCell - 1)) * kShadowCnt) | kShadowBeg; +uptr HiAppMemBeg(void) { + return MappingArchImpl<MAPPING_HI_APP_BEG>(); +} +ALWAYS_INLINE +uptr HiAppMemEnd(void) { + return MappingArchImpl<MAPPING_HI_APP_END>(); } ALWAYS_INLINE -u32 *MemToMeta(uptr x) { - DCHECK(IsAppMem(x)); - return (u32*)(((x & ~(kMetaShadowCell - 1)) / \ - kMetaShadowCell * kMetaShadowSize) | kMetaShadowBeg); +uptr VdsoBeg(void) { + return MappingArchImpl<MAPPING_VDSO_BEG>(); } +#else + ALWAYS_INLINE -uptr ShadowToMem(uptr s) { - CHECK(IsShadowMem(s)); - return (s & ~kShadowBeg) / kShadowCnt; +uptr AppMemBeg(void) { + return MappingArchImpl<MAPPING_APP_BEG>(); +} +ALWAYS_INLINE +uptr AppMemEnd(void) { + return MappingArchImpl<MAPPING_APP_END>(); } -static USED uptr UserRegions[] = { - kAppMemBeg, kAppMemEnd, -}; +#endif -#elif defined(SANITIZER_GO) && SANITIZER_WINDOWS +static inline +bool GetUserRegion(int i, uptr *start, uptr *end) { + switch (i) { + default: + return false; +#if !SANITIZER_GO + case 0: + *start = LoAppMemBeg(); + *end = LoAppMemEnd(); + return true; + case 1: + *start = HiAppMemBeg(); + *end = HiAppMemEnd(); + return true; + case 2: + *start = HeapMemBeg(); + *end = HeapMemEnd(); + return true; +# ifdef TSAN_MID_APP_RANGE + case 3: + *start = MidAppMemBeg(); + *end = MidAppMemEnd(); + return true; +# endif +#else + case 0: + *start = AppMemBeg(); + *end = AppMemEnd(); + return true; +#endif + } +} -/* Go on windows -0000 0000 1000 - 0000 1000 0000: executable -0000 1000 0000 - 00f8 0000 0000: - -00c0 0000 0000 - 00e0 0000 0000: heap -00e0 0000 0000 - 0100 0000 0000: - -0100 0000 0000 - 0500 0000 0000: shadow -0500 0000 0000 - 0560 0000 0000: - -0560 0000 0000 - 0760 0000 0000: traces -0760 0000 0000 - 07d0 0000 0000: metainfo (memory blocks and sync objects) -07d0 0000 0000 - 8000 0000 0000: - -*/ +ALWAYS_INLINE +uptr ShadowBeg(void) { + return MappingArchImpl<MAPPING_SHADOW_BEG>(); +} +ALWAYS_INLINE +uptr ShadowEnd(void) { + return MappingArchImpl<MAPPING_SHADOW_END>(); +} -const uptr kMetaShadowBeg = 0x076000000000ull; -const uptr kMetaShadowEnd = 0x07d000000000ull; -const uptr kTraceMemBeg = 0x056000000000ull; -const uptr kTraceMemEnd = 0x076000000000ull; -const uptr kShadowBeg = 0x010000000000ull; -const uptr kShadowEnd = 0x050000000000ull; -const uptr kAppMemBeg = 0x000000001000ull; -const uptr kAppMemEnd = 0x00e000000000ull; +ALWAYS_INLINE +uptr MetaShadowBeg(void) { + return MappingArchImpl<MAPPING_META_SHADOW_BEG>(); +} +ALWAYS_INLINE +uptr MetaShadowEnd(void) { + return MappingArchImpl<MAPPING_META_SHADOW_END>(); +} + +ALWAYS_INLINE +uptr TraceMemBeg(void) { + return MappingArchImpl<MAPPING_TRACE_BEG>(); +} +ALWAYS_INLINE +uptr TraceMemEnd(void) { + return MappingArchImpl<MAPPING_TRACE_END>(); +} + + +template<typename Mapping> +bool IsAppMemImpl(uptr mem) { +#if !SANITIZER_GO + return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) || +# ifdef TSAN_MID_APP_RANGE + (mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) || +# endif + (mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) || + (mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd); +#else + return mem >= Mapping::kAppMemBeg && mem < Mapping::kAppMemEnd; +#endif +} ALWAYS_INLINE bool IsAppMem(uptr mem) { - return mem >= kAppMemBeg && mem < kAppMemEnd; +#ifdef __aarch64__ + switch (vmaSize) { + case 39: return IsAppMemImpl<Mapping39>(mem); + case 42: return IsAppMemImpl<Mapping42>(mem); + case 48: return IsAppMemImpl<Mapping48>(mem); + } + DCHECK(0); + return false; +#elif defined(__powerpc64__) + if (vmaSize == 44) + return IsAppMemImpl<Mapping44>(mem); + else + return IsAppMemImpl<Mapping46>(mem); + DCHECK(0); +#else + return IsAppMemImpl<Mapping>(mem); +#endif +} + + +template<typename Mapping> +bool IsShadowMemImpl(uptr mem) { + return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd; } ALWAYS_INLINE bool IsShadowMem(uptr mem) { - return mem >= kShadowBeg && mem <= kShadowEnd; +#ifdef __aarch64__ + switch (vmaSize) { + case 39: return IsShadowMemImpl<Mapping39>(mem); + case 42: return IsShadowMemImpl<Mapping42>(mem); + case 48: return IsShadowMemImpl<Mapping48>(mem); + } + DCHECK(0); + return false; +#elif defined(__powerpc64__) + if (vmaSize == 44) + return IsShadowMemImpl<Mapping44>(mem); + else + return IsShadowMemImpl<Mapping46>(mem); + DCHECK(0); +#else + return IsShadowMemImpl<Mapping>(mem); +#endif +} + + +template<typename Mapping> +bool IsMetaMemImpl(uptr mem) { + return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd; } ALWAYS_INLINE bool IsMetaMem(uptr mem) { - return mem >= kMetaShadowBeg && mem <= kMetaShadowEnd; +#ifdef __aarch64__ + switch (vmaSize) { + case 39: return IsMetaMemImpl<Mapping39>(mem); + case 42: return IsMetaMemImpl<Mapping42>(mem); + case 48: return IsMetaMemImpl<Mapping48>(mem); + } + DCHECK(0); + return false; +#elif defined(__powerpc64__) + if (vmaSize == 44) + return IsMetaMemImpl<Mapping44>(mem); + else + return IsMetaMemImpl<Mapping46>(mem); + DCHECK(0); +#else + return IsMetaMemImpl<Mapping>(mem); +#endif } -ALWAYS_INLINE -uptr MemToShadow(uptr x) { + +template<typename Mapping> +uptr MemToShadowImpl(uptr x) { DCHECK(IsAppMem(x)); - return ((x & ~(kShadowCell - 1)) * kShadowCnt) + kShadowBeg; +#if !SANITIZER_GO + return (((x) & ~(Mapping::kAppMemMsk | (kShadowCell - 1))) + ^ Mapping::kAppMemXor) * kShadowCnt; +#else +# ifndef SANITIZER_WINDOWS + return ((x & ~(kShadowCell - 1)) * kShadowCnt) | Mapping::kShadowBeg; +# else + return ((x & ~(kShadowCell - 1)) * kShadowCnt) + Mapping::kShadowBeg; +# endif +#endif } ALWAYS_INLINE -u32 *MemToMeta(uptr x) { +uptr MemToShadow(uptr x) { +#ifdef __aarch64__ + switch (vmaSize) { + case 39: return MemToShadowImpl<Mapping39>(x); + case 42: return MemToShadowImpl<Mapping42>(x); + case 48: return MemToShadowImpl<Mapping48>(x); + } + DCHECK(0); + return 0; +#elif defined(__powerpc64__) + if (vmaSize == 44) + return MemToShadowImpl<Mapping44>(x); + else + return MemToShadowImpl<Mapping46>(x); + DCHECK(0); +#else + return MemToShadowImpl<Mapping>(x); +#endif +} + + +template<typename Mapping> +u32 *MemToMetaImpl(uptr x) { DCHECK(IsAppMem(x)); +#if !SANITIZER_GO + return (u32*)(((((x) & ~(Mapping::kAppMemMsk | (kMetaShadowCell - 1)))) / + kMetaShadowCell * kMetaShadowSize) | Mapping::kMetaShadowBeg); +#else +# ifndef SANITIZER_WINDOWS return (u32*)(((x & ~(kMetaShadowCell - 1)) / \ - kMetaShadowCell * kMetaShadowSize) | kMetaShadowBeg); + kMetaShadowCell * kMetaShadowSize) | Mapping::kMetaShadowBeg); +# else + return (u32*)(((x & ~(kMetaShadowCell - 1)) / \ + kMetaShadowCell * kMetaShadowSize) + Mapping::kMetaShadowBeg); +# endif +#endif } ALWAYS_INLINE -uptr ShadowToMem(uptr s) { - CHECK(IsShadowMem(s)); - // FIXME(dvyukov): this is most likely wrong as the mapping is not bijection. - return (s - kShadowBeg) / kShadowCnt; +u32 *MemToMeta(uptr x) { +#ifdef __aarch64__ + switch (vmaSize) { + case 39: return MemToMetaImpl<Mapping39>(x); + case 42: return MemToMetaImpl<Mapping42>(x); + case 48: return MemToMetaImpl<Mapping48>(x); + } + DCHECK(0); + return 0; +#elif defined(__powerpc64__) + if (vmaSize == 44) + return MemToMetaImpl<Mapping44>(x); + else + return MemToMetaImpl<Mapping46>(x); + DCHECK(0); +#else + return MemToMetaImpl<Mapping>(x); +#endif } -static USED uptr UserRegions[] = { - kAppMemBeg, kAppMemEnd, -}; +template<typename Mapping> +uptr ShadowToMemImpl(uptr s) { + DCHECK(IsShadowMem(s)); +#if !SANITIZER_GO + // The shadow mapping is non-linear and we've lost some bits, so we don't have + // an easy way to restore the original app address. But the mapping is a + // bijection, so we try to restore the address as belonging to low/mid/high + // range consecutively and see if shadow->app->shadow mapping gives us the + // same address. + uptr p = (s / kShadowCnt) ^ Mapping::kAppMemXor; + if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd && + MemToShadow(p) == s) + return p; +# ifdef TSAN_MID_APP_RANGE + p = ((s / kShadowCnt) ^ Mapping::kAppMemXor) + + (Mapping::kMidAppMemBeg & Mapping::kAppMemMsk); + if (p >= Mapping::kMidAppMemBeg && p < Mapping::kMidAppMemEnd && + MemToShadow(p) == s) + return p; +# endif + return ((s / kShadowCnt) ^ Mapping::kAppMemXor) | Mapping::kAppMemMsk; +#else // #if !SANITIZER_GO +# ifndef SANITIZER_WINDOWS + return (s & ~Mapping::kShadowBeg) / kShadowCnt; +# else + return (s - Mapping::kShadowBeg) / kShadowCnt; +# endif // SANITIZER_WINDOWS +#endif +} + +ALWAYS_INLINE +uptr ShadowToMem(uptr s) { +#ifdef __aarch64__ + switch (vmaSize) { + case 39: return ShadowToMemImpl<Mapping39>(s); + case 42: return ShadowToMemImpl<Mapping42>(s); + case 48: return ShadowToMemImpl<Mapping48>(s); + } + DCHECK(0); + return 0; +#elif defined(__powerpc64__) + if (vmaSize == 44) + return ShadowToMemImpl<Mapping44>(s); + else + return ShadowToMemImpl<Mapping46>(s); + DCHECK(0); #else -# error "Unknown platform" + return ShadowToMemImpl<Mapping>(s); #endif +} + + // The additional page is to catch shadow stack overflow as paging fault. // Windows wants 64K alignment for mmaps. const uptr kTotalTraceSize = (kTraceSize * sizeof(Event) + sizeof(Trace) + (64 << 10) + (64 << 10) - 1) & ~((64 << 10) - 1); -uptr ALWAYS_INLINE GetThreadTrace(int tid) { - uptr p = kTraceMemBeg + (uptr)tid * kTotalTraceSize; - DCHECK_LT(p, kTraceMemEnd); +template<typename Mapping> +uptr GetThreadTraceImpl(int tid) { + uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize; + DCHECK_LT(p, Mapping::kTraceMemEnd); return p; } -uptr ALWAYS_INLINE GetThreadTraceHeader(int tid) { - uptr p = kTraceMemBeg + (uptr)tid * kTotalTraceSize +ALWAYS_INLINE +uptr GetThreadTrace(int tid) { +#ifdef __aarch64__ + switch (vmaSize) { + case 39: return GetThreadTraceImpl<Mapping39>(tid); + case 42: return GetThreadTraceImpl<Mapping42>(tid); + case 48: return GetThreadTraceImpl<Mapping48>(tid); + } + DCHECK(0); + return 0; +#elif defined(__powerpc64__) + if (vmaSize == 44) + return GetThreadTraceImpl<Mapping44>(tid); + else + return GetThreadTraceImpl<Mapping46>(tid); + DCHECK(0); +#else + return GetThreadTraceImpl<Mapping>(tid); +#endif +} + + +template<typename Mapping> +uptr GetThreadTraceHeaderImpl(int tid) { + uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize + kTraceSize * sizeof(Event); - DCHECK_LT(p, kTraceMemEnd); + DCHECK_LT(p, Mapping::kTraceMemEnd); return p; } +ALWAYS_INLINE +uptr GetThreadTraceHeader(int tid) { +#ifdef __aarch64__ + switch (vmaSize) { + case 39: return GetThreadTraceHeaderImpl<Mapping39>(tid); + case 42: return GetThreadTraceHeaderImpl<Mapping42>(tid); + case 48: return GetThreadTraceHeaderImpl<Mapping48>(tid); + } + DCHECK(0); + return 0; +#elif defined(__powerpc64__) + if (vmaSize == 44) + return GetThreadTraceHeaderImpl<Mapping44>(tid); + else + return GetThreadTraceHeaderImpl<Mapping46>(tid); + DCHECK(0); +#else + return GetThreadTraceHeaderImpl<Mapping>(tid); +#endif +} + void InitializePlatform(); +void InitializePlatformEarly(); void CheckAndProtect(); void InitializeShadowMemoryPlatform(); void FlushShadowMemory(); void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive); - -// Says whether the addr relates to a global var. -// Guesses with high probability, may yield both false positives and negatives. -bool IsGlobalVar(uptr addr); int ExtractResolvFDs(void *state, int *fds, int nfd); int ExtractRecvmsgFDs(void *msg, int *fds, int nfd); diff --git a/libsanitizer/tsan/tsan_platform_linux.cc b/libsanitizer/tsan/tsan_platform_linux.cc index 09cec5fdffda..2ed5718a12e3 100644 --- a/libsanitizer/tsan/tsan_platform_linux.cc +++ b/libsanitizer/tsan/tsan_platform_linux.cc @@ -16,6 +16,8 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_linux.h" +#include "sanitizer_common/sanitizer_platform_limits_posix.h" #include "sanitizer_common/sanitizer_posix.h" #include "sanitizer_common/sanitizer_procmaps.h" #include "sanitizer_common/sanitizer_stoptheworld.h" @@ -32,6 +34,10 @@ #include <string.h> #include <stdarg.h> #include <sys/mman.h> +#if SANITIZER_LINUX +#include <sys/personality.h> +#include <setjmp.h> +#endif #include <sys/syscall.h> #include <sys/socket.h> #include <sys/time.h> @@ -66,8 +72,10 @@ void InitializeGuardPtr() __attribute__((visibility("hidden"))); namespace __tsan { -static uptr g_data_start; -static uptr g_data_end; +#ifdef TSAN_RUNTIME_VMA +// Runtime detected VMA size. +uptr vmaSize; +#endif enum { MemTotal = 0, @@ -84,29 +92,30 @@ enum { void FillProfileCallback(uptr p, uptr rss, bool file, uptr *mem, uptr stats_size) { mem[MemTotal] += rss; - if (p >= kShadowBeg && p < kShadowEnd) + if (p >= ShadowBeg() && p < ShadowEnd()) mem[MemShadow] += rss; - else if (p >= kMetaShadowBeg && p < kMetaShadowEnd) + else if (p >= MetaShadowBeg() && p < MetaShadowEnd()) mem[MemMeta] += rss; -#ifndef SANITIZER_GO - else if (p >= kHeapMemBeg && p < kHeapMemEnd) +#if !SANITIZER_GO + else if (p >= HeapMemBeg() && p < HeapMemEnd()) mem[MemHeap] += rss; - else if (p >= kLoAppMemBeg && p < kLoAppMemEnd) + else if (p >= LoAppMemBeg() && p < LoAppMemEnd()) mem[file ? MemFile : MemMmap] += rss; - else if (p >= kHiAppMemBeg && p < kHiAppMemEnd) + else if (p >= HiAppMemBeg() && p < HiAppMemEnd()) mem[file ? MemFile : MemMmap] += rss; #else - else if (p >= kAppMemBeg && p < kAppMemEnd) + else if (p >= AppMemBeg() && p < AppMemEnd()) mem[file ? MemFile : MemMmap] += rss; #endif - else if (p >= kTraceMemBeg && p < kTraceMemEnd) + else if (p >= TraceMemBeg() && p < TraceMemEnd()) mem[MemTrace] += rss; else mem[MemOther] += rss; } void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) { - uptr mem[MemCount] = {}; + uptr mem[MemCount]; + internal_memset(mem, 0, sizeof(mem[0]) * MemCount); __sanitizer::GetMemoryProfile(FillProfileCallback, mem, 7); StackDepotStats *stacks = StackDepotGetStats(); internal_snprintf(buf, buf_size, @@ -123,7 +132,7 @@ void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) { void FlushShadowMemoryCallback( const SuspendedThreadsList &suspended_threads_list, void *argument) { - FlushUnneededShadowMemory(kShadowBeg, kShadowEnd - kShadowBeg); + ReleaseMemoryToOS(ShadowBeg(), ShadowEnd() - ShadowBeg()); } #endif @@ -133,7 +142,7 @@ void FlushShadowMemory() { #endif } -#ifndef SANITIZER_GO +#if !SANITIZER_GO // Mark shadow for .rodata sections with the special kShadowRodata marker. // Accesses to .rodata can't race, so this saves time, memory and trace space. static void MapRodata() { @@ -195,55 +204,35 @@ void InitializeShadowMemoryPlatform() { MapRodata(); } -static void InitDataSeg() { - MemoryMappingLayout proc_maps(true); - uptr start, end, offset; - char name[128]; -#if SANITIZER_FREEBSD - // On FreeBSD BSS is usually the last block allocated within the - // low range and heap is the last block allocated within the range - // 0x800000000-0x8ffffffff. - while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name), - /*protection*/ 0)) { - DPrintf("%p-%p %p %s\n", start, end, offset, name); - if ((start & 0xffff00000000ULL) == 0 && (end & 0xffff00000000ULL) == 0 && - name[0] == '\0') { - g_data_start = start; - g_data_end = end; - } +#endif // #if !SANITIZER_GO + +void InitializePlatformEarly() { +#ifdef TSAN_RUNTIME_VMA + vmaSize = + (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1); +#if defined(__aarch64__) + if (vmaSize != 39 && vmaSize != 42 && vmaSize != 48) { + Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); + Printf("FATAL: Found %d - Supported 39, 42 and 48\n", vmaSize); + Die(); } -#else - bool prev_is_data = false; - while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name), - /*protection*/ 0)) { - DPrintf("%p-%p %p %s\n", start, end, offset, name); - bool is_data = offset != 0 && name[0] != 0; - // BSS may get merged with [heap] in /proc/self/maps. This is not very - // reliable. - bool is_bss = offset == 0 && - (name[0] == 0 || internal_strcmp(name, "[heap]") == 0) && prev_is_data; - if (g_data_start == 0 && is_data) - g_data_start = start; - if (is_bss) - g_data_end = end; - prev_is_data = is_data; +#elif defined(__powerpc64__) + if (vmaSize != 44 && vmaSize != 46) { + Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); + Printf("FATAL: Found %d - Supported 44 and 46\n", vmaSize); + Die(); } #endif - DPrintf("guessed data_start=%p data_end=%p\n", g_data_start, g_data_end); - CHECK_LT(g_data_start, g_data_end); - CHECK_GE((uptr)&g_data_start, g_data_start); - CHECK_LT((uptr)&g_data_start, g_data_end); +#endif } -#endif // #ifndef SANITIZER_GO - void InitializePlatform() { DisableCoreDumperIfNecessary(); // Go maps shadow memory lazily and works fine with limited address space. // Unlimited stack is not a problem as well, because the executable // is not compiled with -pie. - if (kCppMode) { + if (!SANITIZER_GO) { bool reexec = false; // TSan doesn't play well with unlimited stack size (as stack // overlaps with shadow memory). If we detect unlimited stack size, @@ -266,6 +255,18 @@ void InitializePlatform() { reexec = true; } #if SANITIZER_LINUX && defined(__aarch64__) + // After patch "arm64: mm: support ARCH_MMAP_RND_BITS." is introduced in + // linux kernel, the random gap between stack and mapped area is increased + // from 128M to 36G on 39-bit aarch64. As it is almost impossible to cover + // this big range, we should disable randomized virtual space on aarch64. + int old_personality = personality(0xffffffff); + if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) { + VReport(1, "WARNING: Program is run with randomized virtual address " + "space, which wouldn't work with ThreadSanitizer.\n" + "Re-execing with fixed virtual address space.\n"); + CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1); + reexec = true; + } // Initialize the guard pointer used in {sig}{set,long}jump. InitializeGuardPtr(); #endif @@ -273,23 +274,18 @@ void InitializePlatform() { ReExec(); } -#ifndef SANITIZER_GO +#if !SANITIZER_GO CheckAndProtect(); InitTlsSize(); - InitDataSeg(); #endif } -bool IsGlobalVar(uptr addr) { - return g_data_start && addr >= g_data_start && addr < g_data_end; -} - -#ifndef SANITIZER_GO +#if !SANITIZER_GO // Extract file descriptors passed to glibc internal __res_iclose function. // This is required to properly "close" the fds, because we do not see internal // closes within glibc. The code is a pure hack. int ExtractResolvFDs(void *state, int *fds, int nfd) { -#if SANITIZER_LINUX +#if SANITIZER_LINUX && !SANITIZER_ANDROID int cnt = 0; __res_state *statp = (__res_state*)state; for (int i = 0; i < MAXNS && cnt < nfd; i++) { @@ -337,10 +333,73 @@ int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m, } #endif -#ifndef SANITIZER_GO +#if !SANITIZER_GO void ReplaceSystemMalloc() { } #endif +#if !SANITIZER_GO +#if SANITIZER_ANDROID + +#if defined(__aarch64__) +# define __get_tls() \ + ({ void** __val; __asm__("mrs %0, tpidr_el0" : "=r"(__val)); __val; }) +#elif defined(__x86_64__) +# define __get_tls() \ + ({ void** __val; __asm__("mov %%fs:0, %0" : "=r"(__val)); __val; }) +#else +#error unsupported architecture +#endif + +// On Android, __thread is not supported. So we store the pointer to ThreadState +// in TLS_SLOT_TSAN, which is the tls slot allocated by Android bionic for tsan. +static const int TLS_SLOT_TSAN = 8; +// On Android, one thread can call intercepted functions after +// DestroyThreadState(), so add a fake thread state for "dead" threads. +static ThreadState *dead_thread_state = nullptr; + +ThreadState *cur_thread() { + ThreadState* thr = (ThreadState*)__get_tls()[TLS_SLOT_TSAN]; + if (thr == nullptr) { + __sanitizer_sigset_t emptyset; + internal_sigfillset(&emptyset); + __sanitizer_sigset_t oldset; + CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset)); + thr = reinterpret_cast<ThreadState*>(__get_tls()[TLS_SLOT_TSAN]); + if (thr == nullptr) { + thr = reinterpret_cast<ThreadState*>(MmapOrDie(sizeof(ThreadState), + "ThreadState")); + __get_tls()[TLS_SLOT_TSAN] = thr; + if (dead_thread_state == nullptr) { + dead_thread_state = reinterpret_cast<ThreadState*>( + MmapOrDie(sizeof(ThreadState), "ThreadState")); + dead_thread_state->fast_state.SetIgnoreBit(); + dead_thread_state->ignore_interceptors = 1; + dead_thread_state->is_dead = true; + *const_cast<int*>(&dead_thread_state->tid) = -1; + CHECK_EQ(0, internal_mprotect(dead_thread_state, sizeof(ThreadState), + PROT_READ)); + } + } + CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr)); + } + return thr; +} + +void cur_thread_finalize() { + __sanitizer_sigset_t emptyset; + internal_sigfillset(&emptyset); + __sanitizer_sigset_t oldset; + CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset)); + ThreadState* thr = (ThreadState*)__get_tls()[TLS_SLOT_TSAN]; + if (thr != dead_thread_state) { + __get_tls()[TLS_SLOT_TSAN] = dead_thread_state; + UnmapOrDie(thr, sizeof(ThreadState)); + } + CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr)); +} +#endif // SANITIZER_ANDROID +#endif // if !SANITIZER_GO + } // namespace __tsan #endif // SANITIZER_LINUX || SANITIZER_FREEBSD diff --git a/libsanitizer/tsan/tsan_platform_mac.cc b/libsanitizer/tsan/tsan_platform_mac.cc index e1405ffeabf1..ff5131e75bb3 100644 --- a/libsanitizer/tsan/tsan_platform_mac.cc +++ b/libsanitizer/tsan/tsan_platform_mac.cc @@ -40,7 +40,7 @@ namespace __tsan { -#ifndef SANITIZER_GO +#if !SANITIZER_GO static void *SignalSafeGetOrAllocate(uptr *dst, uptr size) { atomic_uintptr_t *a = (atomic_uintptr_t *)dst; void *val = (void *)atomic_load_relaxed(a); @@ -65,20 +65,18 @@ static void *SignalSafeGetOrAllocate(uptr *dst, uptr size) { // when TLVs are not accessible (early process startup, thread cleanup, ...). // The following provides a "poor man's TLV" implementation, where we use the // shadow memory of the pointer returned by pthread_self() to store a pointer to -// the ThreadState object. The main thread's ThreadState pointer is stored -// separately in a static variable, because we need to access it even before the +// the ThreadState object. The main thread's ThreadState is stored separately +// in a static variable, because we need to access it even before the // shadow memory is set up. static uptr main_thread_identity = 0; -static ThreadState *main_thread_state = nullptr; +ALIGNED(64) static char main_thread_state[sizeof(ThreadState)]; ThreadState *cur_thread() { - ThreadState **fake_tls; uptr thread_identity = (uptr)pthread_self(); if (thread_identity == main_thread_identity || main_thread_identity == 0) { - fake_tls = &main_thread_state; - } else { - fake_tls = (ThreadState **)MemToShadow(thread_identity); + return (ThreadState *)&main_thread_state; } + ThreadState **fake_tls = (ThreadState **)MemToShadow(thread_identity); ThreadState *thr = (ThreadState *)SignalSafeGetOrAllocate( (uptr *)fake_tls, sizeof(ThreadState)); return thr; @@ -89,7 +87,11 @@ ThreadState *cur_thread() { // handler will try to access the unmapped ThreadState. void cur_thread_finalize() { uptr thread_identity = (uptr)pthread_self(); - CHECK_NE(thread_identity, main_thread_identity); + if (thread_identity == main_thread_identity) { + // Calling dispatch_main() or xpc_main() actually invokes pthread_exit to + // exit the main thread. Let's keep the main thread's ThreadState. + return; + } ThreadState **fake_tls = (ThreadState **)MemToShadow(thread_identity); internal_munmap(*fake_tls, sizeof(ThreadState)); *fake_tls = nullptr; @@ -106,7 +108,7 @@ void FlushShadowMemory() { void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) { } -#ifndef SANITIZER_GO +#if !SANITIZER_GO void InitializeShadowMemoryPlatform() { } // On OS X, GCD worker threads are created without a call to pthread_create. We @@ -122,23 +124,27 @@ typedef void (*pthread_introspection_hook_t)(unsigned int event, extern "C" pthread_introspection_hook_t pthread_introspection_hook_install( pthread_introspection_hook_t hook); static const uptr PTHREAD_INTROSPECTION_THREAD_CREATE = 1; -static const uptr PTHREAD_INTROSPECTION_THREAD_DESTROY = 4; +static const uptr PTHREAD_INTROSPECTION_THREAD_TERMINATE = 3; static pthread_introspection_hook_t prev_pthread_introspection_hook; static void my_pthread_introspection_hook(unsigned int event, pthread_t thread, void *addr, size_t size) { if (event == PTHREAD_INTROSPECTION_THREAD_CREATE) { if (thread == pthread_self()) { // The current thread is a newly created GCD worker thread. + ThreadState *thr = cur_thread(); + Processor *proc = ProcCreate(); + ProcWire(proc, thr); ThreadState *parent_thread_state = nullptr; // No parent. int tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true); CHECK_NE(tid, 0); - ThreadState *thr = cur_thread(); ThreadStart(thr, tid, GetTid()); } - } else if (event == PTHREAD_INTROSPECTION_THREAD_DESTROY) { - ThreadState *thr = cur_thread(); - if (thr->tctx && thr->tctx->parent_tid == kInvalidTid) { - DestroyThreadState(); + } else if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) { + if (thread == pthread_self()) { + ThreadState *thr = cur_thread(); + if (thr->tctx) { + DestroyThreadState(); + } } } @@ -147,9 +153,12 @@ static void my_pthread_introspection_hook(unsigned int event, pthread_t thread, } #endif +void InitializePlatformEarly() { +} + void InitializePlatform() { DisableCoreDumperIfNecessary(); -#ifndef SANITIZER_GO +#if !SANITIZER_GO CheckAndProtect(); CHECK_EQ(main_thread_identity, 0); @@ -160,7 +169,7 @@ void InitializePlatform() { #endif } -#ifndef SANITIZER_GO +#if !SANITIZER_GO // Note: this function runs with async signals enabled, // so it must not touch any tsan state. int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m, @@ -176,10 +185,6 @@ int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m, } #endif -bool IsGlobalVar(uptr addr) { - return false; -} - } // namespace __tsan #endif // SANITIZER_MAC diff --git a/libsanitizer/tsan/tsan_platform_posix.cc b/libsanitizer/tsan/tsan_platform_posix.cc index 5e3d12e9496e..0a3b61a08dc4 100644 --- a/libsanitizer/tsan/tsan_platform_posix.cc +++ b/libsanitizer/tsan/tsan_platform_posix.cc @@ -21,15 +21,16 @@ namespace __tsan { -#ifndef SANITIZER_GO +#if !SANITIZER_GO void InitializeShadowMemory() { // Map memory shadow. uptr shadow = - (uptr)MmapFixedNoReserve(kShadowBeg, kShadowEnd - kShadowBeg, "shadow"); - if (shadow != kShadowBeg) { + (uptr)MmapFixedNoReserve(ShadowBeg(), ShadowEnd() - ShadowBeg(), + "shadow"); + if (shadow != ShadowBeg()) { Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n"); Printf("FATAL: Make sure to compile with -fPIE and " - "to link with -pie (%p, %p).\n", shadow, kShadowBeg); + "to link with -pie (%p, %p).\n", shadow, ShadowBeg()); Die(); } // This memory range is used for thread stacks and large user mmaps. @@ -44,29 +45,50 @@ void InitializeShadowMemory() { const uptr kMadviseRangeBeg = 0xff00000000ull; const uptr kMadviseRangeSize = 0x0100000000ull; #elif defined(__aarch64__) - const uptr kMadviseRangeBeg = 0x7e00000000ull; - const uptr kMadviseRangeSize = 0x0100000000ull; + uptr kMadviseRangeBeg = 0; + uptr kMadviseRangeSize = 0; + if (vmaSize == 39) { + kMadviseRangeBeg = 0x7d00000000ull; + kMadviseRangeSize = 0x0300000000ull; + } else if (vmaSize == 42) { + kMadviseRangeBeg = 0x3f000000000ull; + kMadviseRangeSize = 0x01000000000ull; + } else { + DCHECK(0); + } +#elif defined(__powerpc64__) + uptr kMadviseRangeBeg = 0; + uptr kMadviseRangeSize = 0; + if (vmaSize == 44) { + kMadviseRangeBeg = 0x0f60000000ull; + kMadviseRangeSize = 0x0010000000ull; + } else if (vmaSize == 46) { + kMadviseRangeBeg = 0x3f0000000000ull; + kMadviseRangeSize = 0x010000000000ull; + } else { + DCHECK(0); + } #endif NoHugePagesInRegion(MemToShadow(kMadviseRangeBeg), kMadviseRangeSize * kShadowMultiplier); // Meta shadow is compressing and we don't flush it, // so it makes sense to mark it as NOHUGEPAGE to not over-allocate memory. // On one program it reduces memory consumption from 5GB to 2.5GB. - NoHugePagesInRegion(kMetaShadowBeg, kMetaShadowEnd - kMetaShadowBeg); + NoHugePagesInRegion(MetaShadowBeg(), MetaShadowEnd() - MetaShadowBeg()); if (common_flags()->use_madv_dontdump) - DontDumpShadowMemory(kShadowBeg, kShadowEnd - kShadowBeg); + DontDumpShadowMemory(ShadowBeg(), ShadowEnd() - ShadowBeg()); DPrintf("memory shadow: %zx-%zx (%zuGB)\n", - kShadowBeg, kShadowEnd, - (kShadowEnd - kShadowBeg) >> 30); + ShadowBeg(), ShadowEnd(), + (ShadowEnd() - ShadowBeg()) >> 30); // Map meta shadow. - uptr meta_size = kMetaShadowEnd - kMetaShadowBeg; + uptr meta_size = MetaShadowEnd() - MetaShadowBeg(); uptr meta = - (uptr)MmapFixedNoReserve(kMetaShadowBeg, meta_size, "meta shadow"); - if (meta != kMetaShadowBeg) { + (uptr)MmapFixedNoReserve(MetaShadowBeg(), meta_size, "meta shadow"); + if (meta != MetaShadowBeg()) { Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n"); Printf("FATAL: Make sure to compile with -fPIE and " - "to link with -pie (%p, %p).\n", meta, kMetaShadowBeg); + "to link with -pie (%p, %p).\n", meta, MetaShadowBeg()); Die(); } if (common_flags()->use_madv_dontdump) @@ -81,7 +103,7 @@ static void ProtectRange(uptr beg, uptr end) { CHECK_LE(beg, end); if (beg == end) return; - if (beg != (uptr)MmapNoAccess(beg, end - beg)) { + if (beg != (uptr)MmapFixedNoAccess(beg, end - beg)) { Printf("FATAL: ThreadSanitizer can not protect [%zx,%zx]\n", beg, end); Printf("FATAL: Make sure you are not using unlimited stack\n"); Die(); @@ -95,25 +117,30 @@ void CheckAndProtect() { while (proc_maps.Next(&p, &end, 0, 0, 0, &prot)) { if (IsAppMem(p)) continue; - if (p >= kHeapMemEnd && + if (p >= HeapMemEnd() && p < HeapEnd()) continue; if (prot == 0) // Zero page or mprotected. continue; - if (p >= kVdsoBeg) // vdso + if (p >= VdsoBeg()) // vdso break; Printf("FATAL: ThreadSanitizer: unexpected memory mapping %p-%p\n", p, end); Die(); } - ProtectRange(kLoAppMemEnd, kShadowBeg); - ProtectRange(kShadowEnd, kMetaShadowBeg); - ProtectRange(kMetaShadowEnd, kTraceMemBeg); + ProtectRange(LoAppMemEnd(), ShadowBeg()); + ProtectRange(ShadowEnd(), MetaShadowBeg()); +#ifdef TSAN_MID_APP_RANGE + ProtectRange(MetaShadowEnd(), MidAppMemBeg()); + ProtectRange(MidAppMemEnd(), TraceMemBeg()); +#else + ProtectRange(MetaShadowEnd(), TraceMemBeg()); +#endif // Memory for traces is mapped lazily in MapThreadTrace. // Protect the whole range for now, so that user does not map something here. - ProtectRange(kTraceMemBeg, kTraceMemEnd); - ProtectRange(kTraceMemEnd, kHeapMemBeg); - ProtectRange(HeapEnd(), kHiAppMemBeg); + ProtectRange(TraceMemBeg(), TraceMemEnd()); + ProtectRange(TraceMemEnd(), HeapMemBeg()); + ProtectRange(HeapEnd(), HiAppMemBeg()); } #endif diff --git a/libsanitizer/tsan/tsan_platform_windows.cc b/libsanitizer/tsan/tsan_platform_windows.cc index ccea22ed66e3..54ec6a655dfc 100644 --- a/libsanitizer/tsan/tsan_platform_windows.cc +++ b/libsanitizer/tsan/tsan_platform_windows.cc @@ -29,6 +29,9 @@ void FlushShadowMemory() { void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) { } +void InitializePlatformEarly() { +} + void InitializePlatform() { } diff --git a/libsanitizer/tsan/tsan_ppc_regs.h b/libsanitizer/tsan/tsan_ppc_regs.h new file mode 100644 index 000000000000..15bd10ad96bc --- /dev/null +++ b/libsanitizer/tsan/tsan_ppc_regs.h @@ -0,0 +1,94 @@ +#define r0 0 +#define r3 3 +#define r4 4 +#define r5 5 +#define r6 6 +#define r7 7 +#define r8 8 +#define r9 9 +#define r10 10 +#define r11 11 +#define r12 12 +#define r13 13 +#define r14 14 +#define r15 15 +#define r16 16 +#define r17 17 +#define r18 18 +#define r19 19 +#define r20 20 +#define r21 21 +#define r22 22 +#define r23 23 +#define r24 24 +#define r25 25 +#define r26 26 +#define r27 27 +#define r28 28 +#define r29 29 +#define r30 30 +#define r31 31 +#define f0 0 +#define f1 1 +#define f2 2 +#define f3 3 +#define f4 4 +#define f5 5 +#define f6 6 +#define f7 7 +#define f8 8 +#define f9 9 +#define f10 10 +#define f11 11 +#define f12 12 +#define f13 13 +#define f14 14 +#define f15 15 +#define f16 16 +#define f17 17 +#define f18 18 +#define f19 19 +#define f20 20 +#define f21 21 +#define f22 22 +#define f23 23 +#define f24 24 +#define f25 25 +#define f26 26 +#define f27 27 +#define f28 28 +#define f29 29 +#define f30 30 +#define f31 31 +#define v0 0 +#define v1 1 +#define v2 2 +#define v3 3 +#define v4 4 +#define v5 5 +#define v6 6 +#define v7 7 +#define v8 8 +#define v9 9 +#define v10 10 +#define v11 11 +#define v12 12 +#define v13 13 +#define v14 14 +#define v15 15 +#define v16 16 +#define v17 17 +#define v18 18 +#define v19 19 +#define v20 20 +#define v21 21 +#define v22 22 +#define v23 23 +#define v24 24 +#define v25 25 +#define v26 26 +#define v27 27 +#define v28 28 +#define v29 29 +#define v30 30 +#define v31 31 diff --git a/libsanitizer/tsan/tsan_preinit.cc b/libsanitizer/tsan/tsan_preinit.cc new file mode 100644 index 000000000000..d5d1659c09b6 --- /dev/null +++ b/libsanitizer/tsan/tsan_preinit.cc @@ -0,0 +1,25 @@ +//===-- tsan_preinit.cc ---------------------------------------------------===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer. +// +// Call __tsan_init at the very early stage of process startup. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_internal_defs.h" +#include "tsan_interface.h" + +#if SANITIZER_CAN_USE_PREINIT_ARRAY + +// The symbol is called __local_tsan_preinit, because it's not intended to be +// exported. +// This code linked into the main executable when -fsanitize=thread is in +// the link flags. It can only use exported interface functions. +__attribute__((section(".preinit_array"), used)) +void (*__local_tsan_preinit)(void) = __tsan_init; + +#endif diff --git a/libsanitizer/tsan/tsan_report.cc b/libsanitizer/tsan/tsan_report.cc index 119b1ec1da9c..ba3e34fbfa50 100644 --- a/libsanitizer/tsan/tsan_report.cc +++ b/libsanitizer/tsan/tsan_report.cc @@ -69,7 +69,7 @@ ReportDesc::~ReportDesc() { // FIXME(dvyukov): it must be leaking a lot of memory. } -#ifndef SANITIZER_GO +#if !SANITIZER_GO const int kThreadBufSize = 32; const char *thread_name(char *buf, int tid) { @@ -94,6 +94,8 @@ static const char *ReportTypeString(ReportType typ) { return "destroy of a locked mutex"; if (typ == ReportTypeMutexDoubleLock) return "double lock of a mutex"; + if (typ == ReportTypeMutexInvalidAccess) + return "use of an invalid mutex (e.g. uninitialized or destroyed)"; if (typ == ReportTypeMutexBadUnlock) return "unlock of an unlocked mutex (or by a wrong thread)"; if (typ == ReportTypeMutexBadReadLock) @@ -232,7 +234,7 @@ static void PrintThread(const ReportThread *rt) { Printf(" '%s'", rt->name); char thrbuf[kThreadBufSize]; Printf(" (tid=%zu, %s) created by %s", - rt->pid, rt->running ? "running" : "finished", + rt->os_id, rt->running ? "running" : "finished", thread_name(thrbuf, rt->parent_tid)); if (rt->stack) Printf(" at:"); @@ -265,10 +267,15 @@ static bool FrameIsInternal(const SymbolizedStack *frame) { if (frame == 0) return false; const char *file = frame->info.file; - return file != 0 && - (internal_strstr(file, "tsan_interceptors.cc") || - internal_strstr(file, "sanitizer_common_interceptors.inc") || - internal_strstr(file, "tsan_interface_")); + const char *module = frame->info.module; + if (file != 0 && + (internal_strstr(file, "tsan_interceptors.cc") || + internal_strstr(file, "sanitizer_common_interceptors.inc") || + internal_strstr(file, "tsan_interface_"))) + return true; + if (module != 0 && (internal_strstr(module, "libclang_rt.tsan_"))) + return true; + return false; } static SymbolizedStack *SkipTsanInternalFrames(SymbolizedStack *frames) { @@ -352,7 +359,7 @@ void PrintReport(const ReportDesc *rep) { Printf("==================\n"); } -#else // #ifndef SANITIZER_GO +#else // #if !SANITIZER_GO const int kMainThreadId = 1; @@ -372,9 +379,9 @@ void PrintStack(const ReportStack *ent) { static void PrintMop(const ReportMop *mop, bool first) { Printf("\n"); - Printf("%s by ", + Printf("%s at %p by ", (first ? (mop->write ? "Write" : "Read") - : (mop->write ? "Previous write" : "Previous read"))); + : (mop->write ? "Previous write" : "Previous read")), mop->addr); if (mop->tid == kMainThreadId) Printf("main goroutine:\n"); else @@ -382,6 +389,31 @@ static void PrintMop(const ReportMop *mop, bool first) { PrintStack(mop->stack); } +static void PrintLocation(const ReportLocation *loc) { + switch (loc->type) { + case ReportLocationHeap: { + Printf("\n"); + Printf("Heap block of size %zu at %p allocated by ", + loc->heap_chunk_size, loc->heap_chunk_start); + if (loc->tid == kMainThreadId) + Printf("main goroutine:\n"); + else + Printf("goroutine %d:\n", loc->tid); + PrintStack(loc->stack); + break; + } + case ReportLocationGlobal: { + Printf("\n"); + Printf("Global var %s of size %zu at %p declared at %s:%zu\n", + loc->global.name, loc->global.size, loc->global.start, + loc->global.file, loc->global.line); + break; + } + default: + break; + } +} + static void PrintThread(const ReportThread *rt) { if (rt->id == kMainThreadId) return; @@ -397,6 +429,8 @@ void PrintReport(const ReportDesc *rep) { Printf("WARNING: DATA RACE"); for (uptr i = 0; i < rep->mops.Size(); i++) PrintMop(rep->mops[i], i == 0); + for (uptr i = 0; i < rep->locs.Size(); i++) + PrintLocation(rep->locs[i]); for (uptr i = 0; i < rep->threads.Size(); i++) PrintThread(rep->threads[i]); } else if (rep->typ == ReportTypeDeadlock) { diff --git a/libsanitizer/tsan/tsan_report.h b/libsanitizer/tsan/tsan_report.h index 68924edb5479..e51ed4f50f8e 100644 --- a/libsanitizer/tsan/tsan_report.h +++ b/libsanitizer/tsan/tsan_report.h @@ -25,6 +25,7 @@ enum ReportType { ReportTypeThreadLeak, ReportTypeMutexDestroyLocked, ReportTypeMutexDoubleLock, + ReportTypeMutexInvalidAccess, ReportTypeMutexBadUnlock, ReportTypeMutexBadReadLock, ReportTypeMutexBadReadUnlock, @@ -84,7 +85,7 @@ struct ReportLocation { struct ReportThread { int id; - uptr pid; + uptr os_id; bool running; char *name; int parent_tid; diff --git a/libsanitizer/tsan/tsan_rtl.cc b/libsanitizer/tsan/tsan_rtl.cc index 4fceca6f41f9..07fa165e939c 100644 --- a/libsanitizer/tsan/tsan_rtl.cc +++ b/libsanitizer/tsan/tsan_rtl.cc @@ -42,7 +42,7 @@ extern "C" void __tsan_resume() { namespace __tsan { -#if !defined(SANITIZER_GO) && !SANITIZER_MAC +#if !SANITIZER_GO && !SANITIZER_MAC THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(64); #endif static char ctx_placeholder[sizeof(Context)] ALIGNED(64); @@ -53,12 +53,12 @@ Context *ctx; bool OnFinalize(bool failed); void OnInitialize(); #else -SANITIZER_INTERFACE_ATTRIBUTE -bool WEAK OnFinalize(bool failed) { +SANITIZER_WEAK_CXX_DEFAULT_IMPL +bool OnFinalize(bool failed) { return failed; } -SANITIZER_INTERFACE_ATTRIBUTE -void WEAK OnInitialize() {} +SANITIZER_WEAK_CXX_DEFAULT_IMPL +void OnInitialize() {} #endif static char thread_registry_placeholder[sizeof(ThreadRegistry)]; @@ -84,7 +84,7 @@ static ThreadContextBase *CreateThreadContext(u32 tid) { return new(mem) ThreadContext(tid); } -#ifndef SANITIZER_GO +#if !SANITIZER_GO static const u32 kThreadQuarantineSize = 16; #else static const u32 kThreadQuarantineSize = 64; @@ -115,7 +115,7 @@ ThreadState::ThreadState(Context *ctx, int tid, int unique_id, u64 epoch, // , ignore_reads_and_writes() // , ignore_interceptors() , clock(tid, reuse_count) -#ifndef SANITIZER_GO +#if !SANITIZER_GO , jmp_bufs(MBlockJmpBuf) #endif , tid(tid) @@ -124,13 +124,13 @@ ThreadState::ThreadState(Context *ctx, int tid, int unique_id, u64 epoch, , stk_size(stk_size) , tls_addr(tls_addr) , tls_size(tls_size) -#ifndef SANITIZER_GO +#if !SANITIZER_GO , last_sleep_clock(tid) #endif { } -#ifndef SANITIZER_GO +#if !SANITIZER_GO static void MemoryProfiler(Context *ctx, fd_t fd, int i) { uptr n_threads; uptr n_running_threads; @@ -233,14 +233,17 @@ static void StopBackgroundThread() { void DontNeedShadowFor(uptr addr, uptr size) { uptr shadow_beg = MemToShadow(addr); uptr shadow_end = MemToShadow(addr + size); - FlushUnneededShadowMemory(shadow_beg, shadow_end - shadow_beg); + ReleaseMemoryToOS(shadow_beg, shadow_end - shadow_beg); } void MapShadow(uptr addr, uptr size) { // Global data is not 64K aligned, but there are no adjacent mappings, // so we can get away with unaligned mapping. // CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment - MmapFixedNoReserve(MemToShadow(addr), size * kShadowMultiplier, "shadow"); + const uptr kPageSize = GetPageSizeCached(); + uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), kPageSize); + uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), kPageSize); + MmapFixedNoReserve(shadow_begin, shadow_end - shadow_begin, "shadow"); // Meta shadow is 2:1, so tread carefully. static bool data_mapped = false; @@ -271,8 +274,8 @@ void MapShadow(uptr addr, uptr size) { void MapThreadTrace(uptr addr, uptr size, const char *name) { DPrintf("#0: Mapping trace at %p-%p(0x%zx)\n", addr, addr + size, size); - CHECK_GE(addr, kTraceMemBeg); - CHECK_LE(addr + size, kTraceMemEnd); + CHECK_GE(addr, TraceMemBeg()); + CHECK_LE(addr + size, TraceMemEnd()); CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment uptr addr1 = (uptr)MmapFixedNoReserve(addr, size, name); if (addr1 != addr) { @@ -283,13 +286,17 @@ void MapThreadTrace(uptr addr, uptr size, const char *name) { } static void CheckShadowMapping() { - for (uptr i = 0; i < ARRAY_SIZE(UserRegions); i += 2) { - const uptr beg = UserRegions[i]; - const uptr end = UserRegions[i + 1]; + uptr beg, end; + for (int i = 0; GetUserRegion(i, &beg, &end); i++) { + // Skip cases for empty regions (heap definition for architectures that + // do not use 64-bit allocator). + if (beg == end) + continue; VPrintf(3, "checking shadow region %p-%p\n", beg, end); + uptr prev = 0; for (uptr p0 = beg; p0 <= end; p0 += (end - beg) / 4) { - for (int x = -1; x <= 1; x++) { - const uptr p = p0 + x; + for (int x = -(int)kShadowCell; x <= (int)kShadowCell; x += kShadowCell) { + const uptr p = RoundDown(p0 + x, kShadowCell); if (p < beg || p >= end) continue; const uptr s = MemToShadow(p); @@ -297,8 +304,18 @@ static void CheckShadowMapping() { VPrintf(3, " checking pointer %p: shadow=%p meta=%p\n", p, s, m); CHECK(IsAppMem(p)); CHECK(IsShadowMem(s)); - CHECK_EQ(p & ~(kShadowCell - 1), ShadowToMem(s)); + CHECK_EQ(p, ShadowToMem(s)); CHECK(IsMetaMem(m)); + if (prev) { + // Ensure that shadow and meta mappings are linear within a single + // user range. Lots of code that processes memory ranges assumes it. + const uptr prev_s = MemToShadow(prev); + const uptr prev_m = (uptr)MemToMeta(prev); + CHECK_EQ(s - prev_s, (p - prev) * kShadowMultiplier); + CHECK_EQ((m - prev_m) / kMetaShadowSize, + (p - prev) / kMetaShadowCell); + } + prev = p; } } } @@ -317,26 +334,35 @@ void Initialize(ThreadState *thr) { SetCheckFailedCallback(TsanCheckFailed); ctx = new(ctx_placeholder) Context; - const char *options = GetEnv(kTsanOptionsEnv); + const char *options = GetEnv(SANITIZER_GO ? "GORACE" : "TSAN_OPTIONS"); CacheBinaryName(); InitializeFlags(&ctx->flags, options); - CheckVMASize(); -#ifndef SANITIZER_GO + AvoidCVE_2016_2143(); + InitializePlatformEarly(); +#if !SANITIZER_GO + // Re-exec ourselves if we need to set additional env or command line args. + MaybeReexec(); + InitializeAllocator(); ReplaceSystemMalloc(); #endif + if (common_flags()->detect_deadlocks) + ctx->dd = DDetector::Create(flags()); + Processor *proc = ProcCreate(); + ProcWire(proc, thr); InitializeInterceptors(); CheckShadowMapping(); InitializePlatform(); InitializeMutex(); InitializeDynamicAnnotations(); -#ifndef SANITIZER_GO +#if !SANITIZER_GO InitializeShadowMemory(); + InitializeAllocatorLate(); #endif // Setup correct file descriptor for error reports. __sanitizer_set_report_path(common_flags()->log_path); InitializeSuppressions(); -#ifndef SANITIZER_GO +#if !SANITIZER_GO InitializeLibIgnore(); Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer); // On MIPS, TSan initialization is run before @@ -347,8 +373,6 @@ void Initialize(ThreadState *thr) { SetSandboxingCallback(StopBackgroundThread); #endif #endif - if (common_flags()->detect_deadlocks) - ctx->dd = DDetector::Create(flags()); VPrintf(1, "***** Running under ThreadSanitizer v2 (pid %d) *****\n", (int)internal_getpid()); @@ -362,6 +386,10 @@ void Initialize(ThreadState *thr) { #endif ctx->initialized = true; +#if !SANITIZER_GO + Symbolizer::LateInitialize(); +#endif + if (flags()->stop_on_start) { Printf("ThreadSanitizer is suspended at startup (pid %d)." " Call __tsan_resume().\n", @@ -384,7 +412,7 @@ int Finalize(ThreadState *thr) { CommonSanitizerReportMutex.Unlock(); ctx->report_mtx.Unlock(); -#ifndef SANITIZER_GO +#if !SANITIZER_GO if (Verbosity()) AllocatorPrintStats(); #endif @@ -392,7 +420,7 @@ int Finalize(ThreadState *thr) { if (ctx->nreported) { failed = true; -#ifndef SANITIZER_GO +#if !SANITIZER_GO Printf("ThreadSanitizer: reported %d warnings\n", ctx->nreported); #else Printf("Found %d data race(s)\n", ctx->nreported); @@ -407,7 +435,7 @@ int Finalize(ThreadState *thr) { if (common_flags()->print_suppressions) PrintMatchedSuppressions(); -#ifndef SANITIZER_GO +#if !SANITIZER_GO if (flags()->print_benign) PrintMatchedBenignRaces(); #endif @@ -422,7 +450,7 @@ int Finalize(ThreadState *thr) { return failed ? common_flags()->exitcode : 0; } -#ifndef SANITIZER_GO +#if !SANITIZER_GO void ForkBefore(ThreadState *thr, uptr pc) { ctx->thread_registry->Lock(); ctx->report_mtx.Lock(); @@ -455,7 +483,7 @@ void ForkChildAfter(ThreadState *thr, uptr pc) { } #endif -#ifdef SANITIZER_GO +#if SANITIZER_GO NOINLINE void GrowShadowStack(ThreadState *thr) { const int sz = thr->shadow_stack_end - thr->shadow_stack; @@ -474,7 +502,7 @@ u32 CurrentStackId(ThreadState *thr, uptr pc) { if (!thr->is_inited) // May happen during bootstrap. return 0; if (pc != 0) { -#ifndef SANITIZER_GO +#if !SANITIZER_GO DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end); #else if (thr->shadow_stack_pos == thr->shadow_stack_end) @@ -520,7 +548,7 @@ uptr TraceParts() { return TraceSize() / kTracePartSize; } -#ifndef SANITIZER_GO +#if !SANITIZER_GO extern "C" void __tsan_trace_switch() { TraceSwitch(cur_thread()); } @@ -553,7 +581,7 @@ void HandleRace(ThreadState *thr, u64 *shadow_mem, thr->racy_state[0] = cur.raw(); thr->racy_state[1] = old.raw(); thr->racy_shadow_addr = shadow_mem; -#ifndef SANITIZER_GO +#if !SANITIZER_GO HACKY_CALL(__tsan_report_race); #else ReportRace(thr); @@ -750,7 +778,7 @@ void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, } #endif - if (kCppMode && *shadow_mem == kShadowRodata) { + if (!SANITIZER_GO && *shadow_mem == kShadowRodata) { // Access to .rodata section, no races here. // Measurements show that it can be 10-20% of all memory accesses. StatInc(thr, StatMop); @@ -837,7 +865,7 @@ static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size, size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1); // UnmapOrDie/MmapFixedNoReserve does not work on Windows, // so we do it only for C/C++. - if (kGoMode || size < common_flags()->clear_shadow_mmap_threshold) { + if (SANITIZER_GO || size < common_flags()->clear_shadow_mmap_threshold) { u64 *p = (u64*)MemToShadow(addr); CHECK(IsShadowMem((uptr)p)); CHECK(IsShadowMem((uptr)(p + size * kShadowCnt / kShadowCell - 1))); @@ -923,7 +951,7 @@ void FuncEntry(ThreadState *thr, uptr pc) { // Shadow stack maintenance can be replaced with // stack unwinding during trace switch (which presumably must be faster). DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack); -#ifndef SANITIZER_GO +#if !SANITIZER_GO DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end); #else if (thr->shadow_stack_pos == thr->shadow_stack_end) @@ -943,7 +971,7 @@ void FuncExit(ThreadState *thr) { } DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack); -#ifndef SANITIZER_GO +#if !SANITIZER_GO DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end); #endif thr->shadow_stack_pos--; @@ -954,7 +982,7 @@ void ThreadIgnoreBegin(ThreadState *thr, uptr pc) { thr->ignore_reads_and_writes++; CHECK_GT(thr->ignore_reads_and_writes, 0); thr->fast_state.SetIgnoreBit(); -#ifndef SANITIZER_GO +#if !SANITIZER_GO if (!ctx->after_multithreaded_fork) thr->mop_ignore_set.Add(CurrentStackId(thr, pc)); #endif @@ -966,7 +994,7 @@ void ThreadIgnoreEnd(ThreadState *thr, uptr pc) { CHECK_GE(thr->ignore_reads_and_writes, 0); if (thr->ignore_reads_and_writes == 0) { thr->fast_state.ClearIgnoreBit(); -#ifndef SANITIZER_GO +#if !SANITIZER_GO thr->mop_ignore_set.Reset(); #endif } @@ -976,7 +1004,7 @@ void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc) { DPrintf("#%d: ThreadIgnoreSyncBegin\n", thr->tid); thr->ignore_sync++; CHECK_GT(thr->ignore_sync, 0); -#ifndef SANITIZER_GO +#if !SANITIZER_GO if (!ctx->after_multithreaded_fork) thr->sync_ignore_set.Add(CurrentStackId(thr, pc)); #endif @@ -986,7 +1014,7 @@ void ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc) { DPrintf("#%d: ThreadIgnoreSyncEnd\n", thr->tid); thr->ignore_sync--; CHECK_GE(thr->ignore_sync, 0); -#ifndef SANITIZER_GO +#if !SANITIZER_GO if (thr->ignore_sync == 0) thr->sync_ignore_set.Reset(); #endif @@ -1010,7 +1038,7 @@ void build_consistency_nostats() {} } // namespace __tsan -#ifndef SANITIZER_GO +#if !SANITIZER_GO // Must be included in this file to make sure everything is inlined. #include "tsan_interface_inl.h" #endif diff --git a/libsanitizer/tsan/tsan_rtl.h b/libsanitizer/tsan/tsan_rtl.h index 12587dd203ef..522c76002a1b 100644 --- a/libsanitizer/tsan/tsan_rtl.h +++ b/libsanitizer/tsan/tsan_rtl.h @@ -50,9 +50,9 @@ namespace __tsan { -#ifndef SANITIZER_GO +#if !SANITIZER_GO struct MapUnmapCallback; -#if defined(__mips64) || defined(__aarch64__) +#if defined(__mips64) || defined(__aarch64__) || defined(__powerpc__) static const uptr kAllocatorSpace = 0; static const uptr kAllocatorSize = SANITIZER_MMAP_RANGE_SIZE; static const uptr kAllocatorRegionSizeLog = 20; @@ -64,8 +64,15 @@ typedef SizeClassAllocator32<kAllocatorSpace, kAllocatorSize, 0, CompactSizeClassMap, kAllocatorRegionSizeLog, ByteMap, MapUnmapCallback> PrimaryAllocator; #else -typedef SizeClassAllocator64<kHeapMemBeg, kHeapMemEnd - kHeapMemBeg, 0, - DefaultSizeClassMap, MapUnmapCallback> PrimaryAllocator; +struct AP64 { // Allocator64 parameters. Deliberately using a short name. + static const uptr kSpaceBeg = Mapping::kHeapMemBeg; + static const uptr kSpaceSize = Mapping::kHeapMemEnd - Mapping::kHeapMemBeg; + static const uptr kMetadataSize = 0; + typedef DefaultSizeClassMap SizeClassMap; + typedef __tsan::MapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; +}; +typedef SizeClassAllocator64<AP64> PrimaryAllocator; #endif typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; typedef LargeMmapAllocator<MapUnmapCallback> SecondaryAllocator; @@ -322,6 +329,36 @@ struct JmpBuf { uptr *shadow_stack_pos; }; +// A Processor represents a physical thread, or a P for Go. +// It is used to store internal resources like allocate cache, and does not +// participate in race-detection logic (invisible to end user). +// In C++ it is tied to an OS thread just like ThreadState, however ideally +// it should be tied to a CPU (this way we will have fewer allocator caches). +// In Go it is tied to a P, so there are significantly fewer Processor's than +// ThreadState's (which are tied to Gs). +// A ThreadState must be wired with a Processor to handle events. +struct Processor { + ThreadState *thr; // currently wired thread, or nullptr +#if !SANITIZER_GO + AllocatorCache alloc_cache; + InternalAllocatorCache internal_alloc_cache; +#endif + DenseSlabAllocCache block_cache; + DenseSlabAllocCache sync_cache; + DenseSlabAllocCache clock_cache; + DDPhysicalThread *dd_pt; +}; + +#if !SANITIZER_GO +// ScopedGlobalProcessor temporary setups a global processor for the current +// thread, if it does not have one. Intended for interceptors that can run +// at the very thread end, when we already destroyed the thread processor. +struct ScopedGlobalProcessor { + ScopedGlobalProcessor(); + ~ScopedGlobalProcessor(); +}; +#endif + // This struct is stored in TLS. struct ThreadState { FastState fast_state; @@ -343,7 +380,7 @@ struct ThreadState { int ignore_reads_and_writes; int ignore_sync; // Go does not support ignores. -#ifndef SANITIZER_GO +#if !SANITIZER_GO IgnoreSet mop_ignore_set; IgnoreSet sync_ignore_set; #endif @@ -356,9 +393,7 @@ struct ThreadState { u64 racy_state[2]; MutexSet mset; ThreadClock clock; -#ifndef SANITIZER_GO - AllocatorCache alloc_cache; - InternalAllocatorCache internal_alloc_cache; +#if !SANITIZER_GO Vector<JmpBuf> jmp_bufs; int ignore_interceptors; #endif @@ -382,17 +417,20 @@ struct ThreadState { #if SANITIZER_DEBUG && !SANITIZER_GO InternalDeadlockDetector internal_deadlock_detector; #endif - DDPhysicalThread *dd_pt; DDLogicalThread *dd_lt; + // Current wired Processor, or nullptr. Required to handle any events. + Processor *proc1; +#if !SANITIZER_GO + Processor *proc() { return proc1; } +#else + Processor *proc(); +#endif + atomic_uintptr_t in_signal_handler; ThreadSignalContext *signal_ctx; - DenseSlabAllocCache block_cache; - DenseSlabAllocCache sync_cache; - DenseSlabAllocCache clock_cache; - -#ifndef SANITIZER_GO +#if !SANITIZER_GO u32 last_sleep_stack_id; ThreadClock last_sleep_clock; #endif @@ -401,14 +439,16 @@ struct ThreadState { // If set, malloc must not be called. int nomalloc; + const ReportDesc *current_report; + explicit ThreadState(Context *ctx, int tid, int unique_id, u64 epoch, unsigned reuse_count, uptr stk_addr, uptr stk_size, uptr tls_addr, uptr tls_size); }; -#ifndef SANITIZER_GO -#if SANITIZER_MAC +#if !SANITIZER_GO +#if SANITIZER_MAC || SANITIZER_ANDROID ThreadState *cur_thread(); void cur_thread_finalize(); #else @@ -418,7 +458,7 @@ INLINE ThreadState *cur_thread() { return reinterpret_cast<ThreadState *>(&cur_thread_placeholder); } INLINE void cur_thread_finalize() { } -#endif // SANITIZER_MAC +#endif // SANITIZER_MAC || SANITIZER_ANDROID #endif // SANITIZER_GO class ThreadContext : public ThreadContextBase { @@ -505,13 +545,13 @@ extern Context *ctx; // The one and the only global runtime context. struct ScopedIgnoreInterceptors { ScopedIgnoreInterceptors() { -#ifndef SANITIZER_GO +#if !SANITIZER_GO cur_thread()->ignore_interceptors++; #endif } ~ScopedIgnoreInterceptors() { -#ifndef SANITIZER_GO +#if !SANITIZER_GO cur_thread()->ignore_interceptors--; #endif } @@ -680,6 +720,11 @@ void ThreadSetName(ThreadState *thr, const char *name); int ThreadCount(ThreadState *thr); void ProcessPendingSignals(ThreadState *thr); +Processor *ProcCreate(); +void ProcDestroy(Processor *proc); +void ProcWire(Processor *proc, ThreadState *thr); +void ProcUnwire(Processor *proc, ThreadState *thr); + void MutexCreate(ThreadState *thr, uptr pc, uptr addr, bool rw, bool recursive, bool linker_init); void MutexDestroy(ThreadState *thr, uptr pc, uptr addr); @@ -690,6 +735,7 @@ void MutexReadLock(ThreadState *thr, uptr pc, uptr addr, bool try_lock = false); void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr); void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr); void MutexRepair(ThreadState *thr, uptr pc, uptr addr); // call on EOWNERDEAD +void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr); void Acquire(ThreadState *thr, uptr pc, uptr addr); // AcquireGlobal synchronizes the current thread with all other threads. @@ -745,7 +791,7 @@ void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs, StatInc(thr, StatEvents); u64 pos = fs.GetTracePos(); if (UNLIKELY((pos % kTracePartSize) == 0)) { -#ifndef SANITIZER_GO +#if !SANITIZER_GO HACKY_CALL(__tsan_trace_switch); #else TraceSwitch(thr); @@ -757,9 +803,9 @@ void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs, *evp = ev; } -#ifndef SANITIZER_GO +#if !SANITIZER_GO uptr ALWAYS_INLINE HeapEnd() { - return kHeapMemEnd + PrimaryAllocator::AdditionalSize(); + return HeapMemEnd() + PrimaryAllocator::AdditionalSize(); } #endif diff --git a/libsanitizer/tsan/tsan_rtl_aarch64.S b/libsanitizer/tsan/tsan_rtl_aarch64.S index ef06f0444ae4..ab5a830d60c0 100644 --- a/libsanitizer/tsan/tsan_rtl_aarch64.S +++ b/libsanitizer/tsan/tsan_rtl_aarch64.S @@ -1,6 +1,4 @@ #include "sanitizer_common/sanitizer_asm.h" - -.section .bss .type __tsan_pointer_chk_guard, %object .size __tsan_pointer_chk_guard, 8 __tsan_pointer_chk_guard: diff --git a/libsanitizer/tsan/tsan_rtl_amd64.S b/libsanitizer/tsan/tsan_rtl_amd64.S index 6df36a500a71..caa832375e52 100644 --- a/libsanitizer/tsan/tsan_rtl_amd64.S +++ b/libsanitizer/tsan/tsan_rtl_amd64.S @@ -1,7 +1,13 @@ #include "sanitizer_common/sanitizer_asm.h" -.hidden __tsan_trace_switch -.globl __tsan_trace_switch_thunk -__tsan_trace_switch_thunk: +#if !defined(__APPLE__) +.section .text +#else +.section __TEXT,__text +#endif + +ASM_HIDDEN(__tsan_trace_switch) +.globl ASM_TSAN_SYMBOL(__tsan_trace_switch_thunk) +ASM_TSAN_SYMBOL(__tsan_trace_switch_thunk): CFI_STARTPROC # Save scratch registers. push %rax @@ -40,7 +46,7 @@ __tsan_trace_switch_thunk: shr $4, %rsp # clear 4 lsb, align to 16 shl $4, %rsp - call __tsan_trace_switch + call ASM_TSAN_SYMBOL(__tsan_trace_switch) # Unalign stack frame back. mov %rbx, %rsp # restore the original rsp @@ -79,9 +85,9 @@ __tsan_trace_switch_thunk: ret CFI_ENDPROC -.hidden __tsan_report_race -.globl __tsan_report_race_thunk -__tsan_report_race_thunk: +ASM_HIDDEN(__tsan_report_race) +.globl ASM_TSAN_SYMBOL(__tsan_report_race_thunk) +ASM_TSAN_SYMBOL(__tsan_report_race_thunk): CFI_STARTPROC # Save scratch registers. push %rax @@ -120,7 +126,7 @@ __tsan_report_race_thunk: shr $4, %rsp # clear 4 lsb, align to 16 shl $4, %rsp - call __tsan_report_race + call ASM_TSAN_SYMBOL(__tsan_report_race) # Unalign stack frame back. mov %rbx, %rsp # restore the original rsp @@ -159,11 +165,13 @@ __tsan_report_race_thunk: ret CFI_ENDPROC -.hidden __tsan_setjmp +ASM_HIDDEN(__tsan_setjmp) +#if !defined(__APPLE__) .comm _ZN14__interception11real_setjmpE,8,8 -.globl setjmp -.type setjmp, @function -setjmp: +#endif +.globl ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp) +ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp)) +ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp): CFI_STARTPROC // save env parameter push %rdi @@ -173,29 +181,38 @@ setjmp: #if defined(__FreeBSD__) lea 8(%rsp), %rdi mov %rdi, %rsi -#else +#elif defined(__APPLE__) + lea 16(%rsp), %rdi + mov %rdi, %rsi +#elif defined(__linux__) lea 16(%rsp), %rdi mov %rdi, %rsi xor %fs:0x30, %rsi // magic mangling of rsp (see libc setjmp) rol $0x11, %rsi +#else +# error "Unknown platform" #endif // call tsan interceptor - call __tsan_setjmp + call ASM_TSAN_SYMBOL(__tsan_setjmp) // restore env parameter pop %rdi CFI_ADJUST_CFA_OFFSET(-8) CFI_RESTORE(%rdi) // tail jump to libc setjmp movl $0, %eax +#if !defined(__APPLE__) movq _ZN14__interception11real_setjmpE@GOTPCREL(%rip), %rdx jmp *(%rdx) +#else + jmp ASM_TSAN_SYMBOL(setjmp) +#endif CFI_ENDPROC -.size setjmp, .-setjmp +ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp)) .comm _ZN14__interception12real__setjmpE,8,8 -.globl _setjmp -.type _setjmp, @function -_setjmp: +.globl ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp) +ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp)) +ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp): CFI_STARTPROC // save env parameter push %rdi @@ -205,29 +222,38 @@ _setjmp: #if defined(__FreeBSD__) lea 8(%rsp), %rdi mov %rdi, %rsi -#else +#elif defined(__APPLE__) + lea 16(%rsp), %rdi + mov %rdi, %rsi +#elif defined(__linux__) lea 16(%rsp), %rdi mov %rdi, %rsi xor %fs:0x30, %rsi // magic mangling of rsp (see libc setjmp) rol $0x11, %rsi +#else +# error "Unknown platform" #endif // call tsan interceptor - call __tsan_setjmp + call ASM_TSAN_SYMBOL(__tsan_setjmp) // restore env parameter pop %rdi CFI_ADJUST_CFA_OFFSET(-8) CFI_RESTORE(%rdi) // tail jump to libc setjmp movl $0, %eax +#if !defined(__APPLE__) movq _ZN14__interception12real__setjmpE@GOTPCREL(%rip), %rdx jmp *(%rdx) +#else + jmp ASM_TSAN_SYMBOL(_setjmp) +#endif CFI_ENDPROC -.size _setjmp, .-_setjmp +ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp)) .comm _ZN14__interception14real_sigsetjmpE,8,8 -.globl sigsetjmp -.type sigsetjmp, @function -sigsetjmp: +.globl ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp) +ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp)) +ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp): CFI_STARTPROC // save env parameter push %rdi @@ -244,14 +270,19 @@ sigsetjmp: #if defined(__FreeBSD__) lea 24(%rsp), %rdi mov %rdi, %rsi -#else +#elif defined(__APPLE__) + lea 32(%rsp), %rdi + mov %rdi, %rsi +#elif defined(__linux__) lea 32(%rsp), %rdi mov %rdi, %rsi xor %fs:0x30, %rsi // magic mangling of rsp (see libc setjmp) rol $0x11, %rsi +#else +# error "Unknown platform" #endif // call tsan interceptor - call __tsan_setjmp + call ASM_TSAN_SYMBOL(__tsan_setjmp) // unalign stack frame add $8, %rsp CFI_ADJUST_CFA_OFFSET(-8) @@ -265,15 +296,20 @@ sigsetjmp: CFI_RESTORE(%rdi) // tail jump to libc sigsetjmp movl $0, %eax +#if !defined(__APPLE__) movq _ZN14__interception14real_sigsetjmpE@GOTPCREL(%rip), %rdx jmp *(%rdx) +#else + jmp ASM_TSAN_SYMBOL(sigsetjmp) +#endif CFI_ENDPROC -.size sigsetjmp, .-sigsetjmp +ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp)) +#if !defined(__APPLE__) .comm _ZN14__interception16real___sigsetjmpE,8,8 -.globl __sigsetjmp -.type __sigsetjmp, @function -__sigsetjmp: +.globl ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp) +ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)) +ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp): CFI_STARTPROC // save env parameter push %rdi @@ -297,7 +333,7 @@ __sigsetjmp: rol $0x11, %rsi #endif // call tsan interceptor - call __tsan_setjmp + call ASM_TSAN_SYMBOL(__tsan_setjmp) // unalign stack frame add $8, %rsp CFI_ADJUST_CFA_OFFSET(-8) @@ -314,7 +350,8 @@ __sigsetjmp: movq _ZN14__interception16real___sigsetjmpE@GOTPCREL(%rip), %rdx jmp *(%rdx) CFI_ENDPROC -.size __sigsetjmp, .-__sigsetjmp +ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)) +#endif // !defined(__APPLE__) #if defined(__FreeBSD__) || defined(__linux__) /* We do not need executable stack. */ diff --git a/libsanitizer/tsan/tsan_rtl_mips64.S b/libsanitizer/tsan/tsan_rtl_mips64.S new file mode 100644 index 000000000000..b1c9d8bb12d8 --- /dev/null +++ b/libsanitizer/tsan/tsan_rtl_mips64.S @@ -0,0 +1,212 @@ +.section .text +.hidden __tsan_setjmp +.comm _ZN14__interception11real_setjmpE,8,8 +.globl setjmp +.type setjmp, @function +setjmp: + + // save env parameters + daddiu $sp,$sp,-40 + sd $s0,32($sp) + sd $ra,24($sp) + sd $fp,16($sp) + sd $gp,8($sp) + + // calculate and save pointer to GOT + lui $gp,%hi(%neg(%gp_rel(setjmp))) + daddu $gp,$gp,$t9 + daddiu $gp,$gp,%lo(%neg(%gp_rel(setjmp))) + move $s0,$gp + + // save jmp_buf + sd $a0,0($sp) + + // obtain $sp + dadd $a0,$zero,$sp + + // call tsan interceptor + jal __tsan_setjmp + daddiu $a1,$a0,40 + + // restore jmp_buf + ld $a0,0($sp) + + // restore gp + move $gp,$s0 + + // load pointer of libc setjmp to t9 + dla $t9,(_ZN14__interception11real_setjmpE) + + // restore env parameters + ld $gp,8($sp) + ld $fp,16($sp) + ld $ra,24($sp) + ld $s0,32($sp) + daddiu $sp,$sp,40 + + // tail jump to libc setjmp + ld $t9,0($t9) + jr $t9 + nop + +.size setjmp, .-setjmp + +.hidden __tsan_setjmp +.globl _setjmp +.comm _ZN14__interception12real__setjmpE,8,8 +.type _setjmp, @function +_setjmp: + + // Save env parameters + daddiu $sp,$sp,-40 + sd $s0,32($sp) + sd $ra,24($sp) + sd $fp,16($sp) + sd $gp,8($sp) + + // calculate and save pointer to GOT + lui $gp,%hi(%neg(%gp_rel(_setjmp))) + daddu $gp,$gp,$t9 + daddiu $gp,$gp,%lo(%neg(%gp_rel(_setjmp))) + move $s0,$gp + + // save jmp_buf + sd $a0,0($sp) + + // obtain $sp + dadd $a0,$zero,$sp + + // call tsan interceptor + jal __tsan_setjmp + daddiu $a1,$a0,40 + + // restore jmp_buf + ld $a0,0($sp) + + // restore gp + move $gp,$s0 + + // load pointer of libc _setjmp to t9 + dla $t9,(_ZN14__interception12real__setjmpE) + + // restore env parameters + ld $gp,8($sp) + ld $fp,16($sp) + ld $ra,24($sp) + ld $s0,32($sp) + daddiu $sp,$sp,40 + + // tail jump to libc _setjmp + ld $t9,0($t9) + jr $t9 + nop + +.size _setjmp, .-_setjmp + +.hidden __tsan_setjmp +.globl sigsetjmp +.comm _ZN14__interception14real_sigsetjmpE,8,8 +.type sigsetjmp, @function +sigsetjmp: + + // Save env parameters + daddiu $sp,$sp,-48 + sd $s0,40($sp) + sd $ra,32($sp) + sd $fp,24($sp) + sd $gp,16($sp) + + // calculate and save pointer to GOT + lui $gp,%hi(%neg(%gp_rel(sigsetjmp))) + daddu $gp,$gp,$t9 + daddiu $gp,$gp,%lo(%neg(%gp_rel(sigsetjmp))) + move $s0,$gp + + // save jmp_buf and savesig + sd $a0,0($sp) + sd $a1,8($sp) + + // obtain $sp + dadd $a0,$zero,$sp + + // call tsan interceptor + jal __tsan_setjmp + daddiu $a1,$a0,48 + + // restore jmp_buf and savesig + ld $a0,0($sp) + ld $a1,8($sp) + + // restore gp + move $gp,$s0 + + // load pointer of libc sigsetjmp to t9 + dla $t9,(_ZN14__interception14real_sigsetjmpE) + + // restore env parameters + ld $gp,16($sp) + ld $fp,24($sp) + ld $ra,32($sp) + ld $s0,40($sp) + daddiu $sp,$sp,48 + + // tail jump to libc sigsetjmp + ld $t9,0($t9) + jr $t9 + nop + +.size sigsetjmp, .-sigsetjmp + +.hidden __tsan_setjmp +.comm _ZN14__interception16real___sigsetjmpE,8,8 +.globl __sigsetjmp +.type __sigsetjmp, @function +__sigsetjmp: + + // Save env parameters + daddiu $sp,$sp,-48 + sd $s0,40($sp) + sd $ra,32($sp) + sd $fp,24($sp) + sd $gp,16($sp) + + // calculate and save pointer to GOT + lui $gp,%hi(%neg(%gp_rel(__sigsetjmp))) + daddu $gp,$gp,$t9 + daddiu $gp,$gp,%lo(%neg(%gp_rel(__sigsetjmp))) + move $s0,$gp + + // save jmp_buf and savesig + sd $a0,0($sp) + sd $a1,8($sp) + + // obtain $sp + dadd $a0,$zero,$sp + + // call tsan interceptor + jal __tsan_setjmp + daddiu $a1,$a0,48 + + // restore jmp_buf and savesig + ld $a0,0($sp) + ld $a1,8($sp) + + // restore gp + move $gp,$s0 + + // load pointer to libc __sigsetjmp in t9 + dla $t9,(_ZN14__interception16real___sigsetjmpE) + + // restore env parameters + ld $gp,16($sp) + ld $fp,24($sp) + ld $ra,32($sp) + ld $s0,40($sp) + daddiu $sp,$sp,48 + + // tail jump to libc __sigsetjmp + ld $t9,0($t9) + jr $t9 + nop + +.size __sigsetjmp, .-__sigsetjmp diff --git a/libsanitizer/tsan/tsan_rtl_mutex.cc b/libsanitizer/tsan/tsan_rtl_mutex.cc index deb7722f521e..e575bbfb7e9e 100644 --- a/libsanitizer/tsan/tsan_rtl_mutex.cc +++ b/libsanitizer/tsan/tsan_rtl_mutex.cc @@ -30,7 +30,7 @@ struct Callback : DDCallback { Callback(ThreadState *thr, uptr pc) : thr(thr) , pc(pc) { - DDCallback::pt = thr->dd_pt; + DDCallback::pt = thr->proc()->dd_pt; DDCallback::lt = thr->dd_lt; } @@ -48,7 +48,7 @@ static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ, uptr addr, u64 mid) { // In Go, these misuses are either impossible, or detected by std lib, // or false positives (e.g. unlock in a different thread). - if (kGoMode) + if (SANITIZER_GO) return; ThreadRegistryLock l(ctx->thread_registry); ScopedReport rep(typ); @@ -74,7 +74,7 @@ void MutexCreate(ThreadState *thr, uptr pc, uptr addr, s->is_rw = rw; s->is_recursive = recursive; s->is_linker_init = linker_init; - if (kCppMode && s->creation_stack_id == 0) + if (!SANITIZER_GO && s->creation_stack_id == 0) s->creation_stack_id = CurrentStackId(thr, pc); s->mtx.Unlock(); } @@ -82,21 +82,14 @@ void MutexCreate(ThreadState *thr, uptr pc, uptr addr, void MutexDestroy(ThreadState *thr, uptr pc, uptr addr) { DPrintf("#%d: MutexDestroy %zx\n", thr->tid, addr); StatInc(thr, StatMutexDestroy); -#ifndef SANITIZER_GO - // Global mutexes not marked as LINKER_INITIALIZED - // cause tons of not interesting reports, so just ignore it. - if (IsGlobalVar(addr)) - return; -#endif - if (IsAppMem(addr)) { - CHECK(!thr->is_freeing); - thr->is_freeing = true; - MemoryWrite(thr, pc, addr, kSizeLog1); - thr->is_freeing = false; - } - SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr); + SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true); if (s == 0) return; + if (s->is_linker_init) { + // Destroy is no-op for linker-initialized mutexes. + s->mtx.Unlock(); + return; + } if (common_flags()->detect_deadlocks) { Callback cb(thr, pc); ctx->dd->MutexDestroy(&cb, &s->dd); @@ -112,7 +105,7 @@ void MutexDestroy(ThreadState *thr, uptr pc, uptr addr) { u64 mid = s->GetId(); u32 last_lock = s->last_lock; if (!unlock_locked) - s->Reset(thr); // must not reset it before the report is printed + s->Reset(thr->proc()); // must not reset it before the report is printed s->mtx.Unlock(); if (unlock_locked) { ThreadRegistryLock l(ctx->thread_registry); @@ -126,15 +119,23 @@ void MutexDestroy(ThreadState *thr, uptr pc, uptr addr) { rep.AddStack(trace, true); rep.AddLocation(addr, 1); OutputReport(thr, rep); - } - if (unlock_locked) { - SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr); + + SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true); if (s != 0) { - s->Reset(thr); + s->Reset(thr->proc()); s->mtx.Unlock(); } } thr->mset.Remove(mid); + // Imitate a memory write to catch unlock-destroy races. + // Do this outside of sync mutex, because it can report a race which locks + // sync mutexes. + if (IsAppMem(addr)) { + CHECK(!thr->is_freeing); + thr->is_freeing = true; + MemoryWrite(thr, pc, addr, kSizeLog1); + thr->is_freeing = false; + } // s will be destroyed and freed in MetaMap::FreeBlock. } @@ -192,7 +193,7 @@ int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, bool all) { TraceAddEvent(thr, thr->fast_state, EventTypeUnlock, s->GetId()); int rec = 0; bool report_bad_unlock = false; - if (kCppMode && (s->recursion == 0 || s->owner_tid != thr->tid)) { + if (!SANITIZER_GO && (s->recursion == 0 || s->owner_tid != thr->tid)) { if (flags()->report_mutex_bugs && !s->is_broken) { s->is_broken = true; report_bad_unlock = true; @@ -348,11 +349,21 @@ void MutexRepair(ThreadState *thr, uptr pc, uptr addr) { s->mtx.Unlock(); } +void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr) { + DPrintf("#%d: MutexInvalidAccess %zx\n", thr->tid, addr); + SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true); + u64 mid = s->GetId(); + s->mtx.Unlock(); + ReportMutexMisuse(thr, pc, ReportTypeMutexInvalidAccess, addr, mid); +} + void Acquire(ThreadState *thr, uptr pc, uptr addr) { DPrintf("#%d: Acquire %zx\n", thr->tid, addr); if (thr->ignore_sync) return; - SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false); + SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, false); + if (!s) + return; AcquireImpl(thr, pc, &s->clock); s->mtx.ReadUnlock(); } @@ -399,7 +410,7 @@ void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) { s->mtx.Unlock(); } -#ifndef SANITIZER_GO +#if !SANITIZER_GO static void UpdateSleepClockCallback(ThreadContextBase *tctx_base, void *arg) { ThreadState *thr = reinterpret_cast<ThreadState*>(arg); ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base); @@ -424,7 +435,7 @@ void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) { if (thr->ignore_sync) return; thr->clock.set(thr->fast_state.epoch()); - thr->clock.acquire(&thr->clock_cache, c); + thr->clock.acquire(&thr->proc()->clock_cache, c); StatInc(thr, StatSyncAcquire); } @@ -433,7 +444,7 @@ void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) { return; thr->clock.set(thr->fast_state.epoch()); thr->fast_synch_epoch = thr->fast_state.epoch(); - thr->clock.release(&thr->clock_cache, c); + thr->clock.release(&thr->proc()->clock_cache, c); StatInc(thr, StatSyncRelease); } @@ -442,7 +453,7 @@ void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c) { return; thr->clock.set(thr->fast_state.epoch()); thr->fast_synch_epoch = thr->fast_state.epoch(); - thr->clock.ReleaseStore(&thr->clock_cache, c); + thr->clock.ReleaseStore(&thr->proc()->clock_cache, c); StatInc(thr, StatSyncRelease); } @@ -451,7 +462,7 @@ void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) { return; thr->clock.set(thr->fast_state.epoch()); thr->fast_synch_epoch = thr->fast_state.epoch(); - thr->clock.acq_rel(&thr->clock_cache, c); + thr->clock.acq_rel(&thr->proc()->clock_cache, c); StatInc(thr, StatSyncAcquire); StatInc(thr, StatSyncRelease); } diff --git a/libsanitizer/tsan/tsan_rtl_ppc64.S b/libsanitizer/tsan/tsan_rtl_ppc64.S new file mode 100644 index 000000000000..81d309fbd2ff --- /dev/null +++ b/libsanitizer/tsan/tsan_rtl_ppc64.S @@ -0,0 +1,286 @@ +#include "tsan_ppc_regs.h" + .hidden __tsan_setjmp + .globl _setjmp + .type _setjmp, @function + .align 4 +#if _CALL_ELF == 2 +_setjmp: +#else + .section ".opd","aw" + .align 3 +_setjmp: + .quad .L._setjmp,.TOC.@tocbase,0 + .previous +#endif +.L._setjmp: + mflr r0 + stdu r1,-48(r1) + std r2,24(r1) + std r3,32(r1) + std r0,40(r1) + // r3 is the original stack pointer. + addi r3,r1,48 + // r4 is the mangled stack pointer (see glibc) + ld r4,-28696(r13) + xor r4,r3,r4 + // Materialize a TOC in case we were called from libc. + // For big-endian, we load the TOC from the OPD. For little- + // endian, we use the .TOC. symbol to find it. + nop + bcl 20,31,0f +0: + mflr r2 +#if _CALL_ELF == 2 + addis r2,r2,.TOC.-0b@ha + addi r2,r2,.TOC.-0b@l +#else + addis r2,r2,_setjmp-0b@ha + addi r2,r2,_setjmp-0b@l + ld r2,8(r2) +#endif + // Call the interceptor. + bl __tsan_setjmp + nop + // Restore regs needed for setjmp. + ld r3,32(r1) + ld r0,40(r1) + // Emulate the real setjmp function. We do this because we can't + // perform a sibcall: The real setjmp function trashes the TOC + // pointer, and with a sibcall we have no way to restore it. + // This way we can make sure our caller's stack pointer and + // link register are saved correctly in the jmpbuf. + ld r6,-28696(r13) + addi r5,r1,48 // original stack ptr of caller + xor r5,r6,r5 + std r5,0(r3) // mangled stack ptr of caller + ld r5,24(r1) + std r5,8(r3) // caller's saved TOC pointer + xor r0,r6,r0 + std r0,16(r3) // caller's mangled return address + mfcr r0 + // Nonvolatiles. + std r14,24(r3) + stfd f14,176(r3) + stw r0,172(r3) // CR + std r15,32(r3) + stfd f15,184(r3) + std r16,40(r3) + stfd f16,192(r3) + std r17,48(r3) + stfd f17,200(r3) + std r18,56(r3) + stfd f18,208(r3) + std r19,64(r3) + stfd f19,216(r3) + std r20,72(r3) + stfd f20,224(r3) + std r21,80(r3) + stfd f21,232(r3) + std r22,88(r3) + stfd f22,240(r3) + std r23,96(r3) + stfd f23,248(r3) + std r24,104(r3) + stfd f24,256(r3) + std r25,112(r3) + stfd f25,264(r3) + std r26,120(r3) + stfd f26,272(r3) + std r27,128(r3) + stfd f27,280(r3) + std r28,136(r3) + stfd f28,288(r3) + std r29,144(r3) + stfd f29,296(r3) + std r30,152(r3) + stfd f30,304(r3) + std r31,160(r3) + stfd f31,312(r3) + addi r5,r3,320 + mfspr r0,256 + stw r0,168(r3) // VRSAVE + addi r6,r5,16 + stvx v20,0,r5 + addi r5,r5,32 + stvx v21,0,r6 + addi r6,r6,32 + stvx v22,0,r5 + addi r5,r5,32 + stvx v23,0,r6 + addi r6,r6,32 + stvx v24,0,r5 + addi r5,r5,32 + stvx v25,0,r6 + addi r6,r6,32 + stvx v26,0,r5 + addi r5,r5,32 + stvx v27,0,r6 + addi r6,r6,32 + stvx v28,0,r5 + addi r5,r5,32 + stvx v29,0,r6 + addi r6,r6,32 + stvx v30,0,r5 + stvx v31,0,r6 + // Clear the "mask-saved" slot. + li r4,0 + stw r4,512(r3) + // Restore TOC, LR, and stack and return to caller. + ld r2,24(r1) + ld r0,40(r1) + addi r1,r1,48 + li r3,0 // This is the setjmp return path + mtlr r0 + blr + .size _setjmp, .-.L._setjmp + + .globl setjmp + .type setjmp, @function + .align 4 +setjmp: + b _setjmp + .size setjmp, .-setjmp + + // sigsetjmp is like setjmp, except that the mask in r4 needs + // to be saved at offset 512 of the jump buffer. + .globl __sigsetjmp + .type __sigsetjmp, @function + .align 4 +#if _CALL_ELF == 2 +__sigsetjmp: +#else + .section ".opd","aw" + .align 3 +__sigsetjmp: + .quad .L.__sigsetjmp,.TOC.@tocbase,0 + .previous +#endif +.L.__sigsetjmp: + mflr r0 + stdu r1,-64(r1) + std r2,24(r1) + std r3,32(r1) + std r4,40(r1) + std r0,48(r1) + // r3 is the original stack pointer. + addi r3,r1,64 + // r4 is the mangled stack pointer (see glibc) + ld r4,-28696(r13) + xor r4,r3,r4 + // Materialize a TOC in case we were called from libc. + // For big-endian, we load the TOC from the OPD. For little- + // endian, we use the .TOC. symbol to find it. + nop + bcl 20,31,1f +1: + mflr r2 +#if _CALL_ELF == 2 + addis r2,r2,.TOC.-1b@ha + addi r2,r2,.TOC.-1b@l +#else + addis r2,r2,_setjmp-1b@ha + addi r2,r2,_setjmp-1b@l + ld r2,8(r2) +#endif + // Call the interceptor. + bl __tsan_setjmp + nop + // Restore regs needed for __sigsetjmp. + ld r3,32(r1) + ld r4,40(r1) + ld r0,48(r1) + // Emulate the real sigsetjmp function. We do this because we can't + // perform a sibcall: The real sigsetjmp function trashes the TOC + // pointer, and with a sibcall we have no way to restore it. + // This way we can make sure our caller's stack pointer and + // link register are saved correctly in the jmpbuf. + ld r6,-28696(r13) + addi r5,r1,64 // original stack ptr of caller + xor r5,r6,r5 + std r5,0(r3) // mangled stack ptr of caller + ld r5,24(r1) + std r5,8(r3) // caller's saved TOC pointer + xor r0,r6,r0 + std r0,16(r3) // caller's mangled return address + mfcr r0 + // Nonvolatiles. + std r14,24(r3) + stfd f14,176(r3) + stw r0,172(r3) // CR + std r15,32(r3) + stfd f15,184(r3) + std r16,40(r3) + stfd f16,192(r3) + std r17,48(r3) + stfd f17,200(r3) + std r18,56(r3) + stfd f18,208(r3) + std r19,64(r3) + stfd f19,216(r3) + std r20,72(r3) + stfd f20,224(r3) + std r21,80(r3) + stfd f21,232(r3) + std r22,88(r3) + stfd f22,240(r3) + std r23,96(r3) + stfd f23,248(r3) + std r24,104(r3) + stfd f24,256(r3) + std r25,112(r3) + stfd f25,264(r3) + std r26,120(r3) + stfd f26,272(r3) + std r27,128(r3) + stfd f27,280(r3) + std r28,136(r3) + stfd f28,288(r3) + std r29,144(r3) + stfd f29,296(r3) + std r30,152(r3) + stfd f30,304(r3) + std r31,160(r3) + stfd f31,312(r3) + addi r5,r3,320 + mfspr r0,256 + stw r0,168(r3) // VRSAVE + addi r6,r5,16 + stvx v20,0,r5 + addi r5,r5,32 + stvx v21,0,r6 + addi r6,r6,32 + stvx v22,0,r5 + addi r5,r5,32 + stvx v23,0,r6 + addi r6,r6,32 + stvx v24,0,r5 + addi r5,r5,32 + stvx v25,0,r6 + addi r6,r6,32 + stvx v26,0,r5 + addi r5,r5,32 + stvx v27,0,r6 + addi r6,r6,32 + stvx v28,0,r5 + addi r5,r5,32 + stvx v29,0,r6 + addi r6,r6,32 + stvx v30,0,r5 + stvx v31,0,r6 + // Save into the "mask-saved" slot. + stw r4,512(r3) + // Restore TOC, LR, and stack and return to caller. + ld r2,24(r1) + ld r0,48(r1) + addi r1,r1,64 + li r3,0 // This is the sigsetjmp return path + mtlr r0 + blr + .size __sigsetjmp, .-.L.__sigsetjmp + + .globl sigsetjmp + .type sigsetjmp, @function + .align 4 +sigsetjmp: + b __sigsetjmp + .size sigsetjmp, .-sigsetjmp diff --git a/libsanitizer/tsan/tsan_rtl_proc.cc b/libsanitizer/tsan/tsan_rtl_proc.cc new file mode 100644 index 000000000000..1b0a9b38938a --- /dev/null +++ b/libsanitizer/tsan/tsan_rtl_proc.cc @@ -0,0 +1,59 @@ +//===-- tsan_rtl_proc.cc ------------------------------------------------===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_placement_new.h" +#include "tsan_rtl.h" +#include "tsan_mman.h" +#include "tsan_flags.h" + +namespace __tsan { + +Processor *ProcCreate() { + void *mem = InternalAlloc(sizeof(Processor)); + internal_memset(mem, 0, sizeof(Processor)); + Processor *proc = new(mem) Processor; + proc->thr = nullptr; +#if !SANITIZER_GO + AllocatorProcStart(proc); +#endif + if (common_flags()->detect_deadlocks) + proc->dd_pt = ctx->dd->CreatePhysicalThread(); + return proc; +} + +void ProcDestroy(Processor *proc) { + CHECK_EQ(proc->thr, nullptr); +#if !SANITIZER_GO + AllocatorProcFinish(proc); +#endif + ctx->clock_alloc.FlushCache(&proc->clock_cache); + ctx->metamap.OnProcIdle(proc); + if (common_flags()->detect_deadlocks) + ctx->dd->DestroyPhysicalThread(proc->dd_pt); + proc->~Processor(); + InternalFree(proc); +} + +void ProcWire(Processor *proc, ThreadState *thr) { + CHECK_EQ(thr->proc1, nullptr); + CHECK_EQ(proc->thr, nullptr); + thr->proc1 = proc; + proc->thr = thr; +} + +void ProcUnwire(Processor *proc, ThreadState *thr) { + CHECK_EQ(thr->proc1, proc); + CHECK_EQ(proc->thr, thr); + thr->proc1 = nullptr; + proc->thr = nullptr; +} + +} // namespace __tsan diff --git a/libsanitizer/tsan/tsan_rtl_report.cc b/libsanitizer/tsan/tsan_rtl_report.cc index d0d1fbaf45ac..8f2882485e61 100644 --- a/libsanitizer/tsan/tsan_rtl_report.cc +++ b/libsanitizer/tsan/tsan_rtl_report.cc @@ -36,6 +36,10 @@ void TsanCheckFailed(const char *file, int line, const char *cond, // on the other hand there is no sense in processing interceptors // since we are going to die soon. ScopedIgnoreInterceptors ignore; +#if !SANITIZER_GO + cur_thread()->ignore_sync++; + cur_thread()->ignore_reads_and_writes++; +#endif Printf("FATAL: ThreadSanitizer CHECK failed: " "%s:%d \"%s\" (0x%zx, 0x%zx)\n", file, line, cond, (uptr)v1, (uptr)v2); @@ -47,13 +51,18 @@ void TsanCheckFailed(const char *file, int line, const char *cond, #ifdef TSAN_EXTERNAL_HOOKS bool OnReport(const ReportDesc *rep, bool suppressed); #else -SANITIZER_INTERFACE_ATTRIBUTE -bool WEAK OnReport(const ReportDesc *rep, bool suppressed) { +SANITIZER_WEAK_CXX_DEFAULT_IMPL +bool OnReport(const ReportDesc *rep, bool suppressed) { (void)rep; return suppressed; } #endif +SANITIZER_WEAK_DEFAULT_IMPL +void __tsan_on_report(const ReportDesc *rep) { + (void)rep; +} + static void StackStripMain(SymbolizedStack *frames) { SymbolizedStack *last_frame = nullptr; SymbolizedStack *last_frame2 = nullptr; @@ -64,7 +73,7 @@ static void StackStripMain(SymbolizedStack *frames) { if (last_frame2 == 0) return; -#ifndef SANITIZER_GO +#if !SANITIZER_GO const char *last = last_frame->info.function; const char *last2 = last_frame2->info.function; // Strip frame above 'main' @@ -184,10 +193,10 @@ void ScopedReport::AddThread(const ThreadContext *tctx, bool suppressable) { return; } void *mem = internal_alloc(MBlockReportThread, sizeof(ReportThread)); - ReportThread *rt = new(mem) ReportThread(); + ReportThread *rt = new(mem) ReportThread; rep_->threads.PushBack(rt); rt->id = tctx->tid; - rt->pid = tctx->os_id; + rt->os_id = tctx->os_id; rt->running = (tctx->status == ThreadStatusRunning); rt->name = internal_strdup(tctx->name); rt->parent_tid = tctx->parent_tid; @@ -197,17 +206,17 @@ void ScopedReport::AddThread(const ThreadContext *tctx, bool suppressable) { rt->stack->suppressable = suppressable; } -#ifndef SANITIZER_GO +#if !SANITIZER_GO +static bool FindThreadByUidLockedCallback(ThreadContextBase *tctx, void *arg) { + int unique_id = *(int *)arg; + return tctx->unique_id == (u32)unique_id; +} + static ThreadContext *FindThreadByUidLocked(int unique_id) { ctx->thread_registry->CheckLocked(); - for (unsigned i = 0; i < kMaxTid; i++) { - ThreadContext *tctx = static_cast<ThreadContext*>( - ctx->thread_registry->GetThreadLocked(i)); - if (tctx && tctx->unique_id == (u32)unique_id) { - return tctx; - } - } - return 0; + return static_cast<ThreadContext *>( + ctx->thread_registry->FindThreadContextLocked( + FindThreadByUidLockedCallback, &unique_id)); } static ThreadContext *FindThreadByTidLocked(int tid) { @@ -242,7 +251,7 @@ ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack) { #endif void ScopedReport::AddThread(int unique_tid, bool suppressable) { -#ifndef SANITIZER_GO +#if !SANITIZER_GO if (const ThreadContext *tctx = FindThreadByUidLocked(unique_tid)) AddThread(tctx, suppressable); #endif @@ -254,7 +263,7 @@ void ScopedReport::AddMutex(const SyncVar *s) { return; } void *mem = internal_alloc(MBlockReportMutex, sizeof(ReportMutex)); - ReportMutex *rm = new(mem) ReportMutex(); + ReportMutex *rm = new(mem) ReportMutex; rep_->mutexes.PushBack(rm); rm->id = s->uid; rm->addr = s->addr; @@ -266,7 +275,7 @@ u64 ScopedReport::AddMutex(u64 id) { u64 uid = 0; u64 mid = id; uptr addr = SyncVar::SplitId(id, &uid); - SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr); + SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true); // Check that the mutex is still alive. // Another mutex can be created at the same address, // so check uid as well. @@ -287,7 +296,7 @@ void ScopedReport::AddDeadMutex(u64 id) { return; } void *mem = internal_alloc(MBlockReportMutex, sizeof(ReportMutex)); - ReportMutex *rm = new(mem) ReportMutex(); + ReportMutex *rm = new(mem) ReportMutex; rep_->mutexes.PushBack(rm); rm->id = id; rm->addr = 0; @@ -298,7 +307,7 @@ void ScopedReport::AddDeadMutex(u64 id) { void ScopedReport::AddLocation(uptr addr, uptr size) { if (addr == 0) return; -#ifndef SANITIZER_GO +#if !SANITIZER_GO int fd = -1; int creat_tid = -1; u32 creat_stack = 0; @@ -340,15 +349,15 @@ void ScopedReport::AddLocation(uptr addr, uptr size) { rep_->locs.PushBack(loc); AddThread(tctx); } +#endif if (ReportLocation *loc = SymbolizeData(addr)) { loc->suppressable = true; rep_->locs.PushBack(loc); return; } -#endif } -#ifndef SANITIZER_GO +#if !SANITIZER_GO void ScopedReport::AddSleep(u32 stack_id) { rep_->sleep = SymbolizeStackId(stack_id); } @@ -490,6 +499,8 @@ bool OutputReport(ThreadState *thr, const ScopedReport &srep) { return false; atomic_store_relaxed(&ctx->last_symbolize_time_ns, NanoTime()); const ReportDesc *rep = srep.GetReport(); + CHECK_EQ(thr->current_report, nullptr); + thr->current_report = rep; Suppression *supp = 0; uptr pc_or_addr = 0; for (uptr i = 0; pc_or_addr == 0 && i < rep->mops.Size(); i++) @@ -510,13 +521,17 @@ bool OutputReport(ThreadState *thr, const ScopedReport &srep) { thr->is_freeing = false; bool suppressed = OnReport(rep, pc_or_addr != 0); thr->is_freeing = old_is_freeing; - if (suppressed) + if (suppressed) { + thr->current_report = nullptr; return false; + } } PrintReport(rep); + __tsan_on_report(rep); ctx->nreported++; if (flags()->halt_on_error) Die(); + thr->current_report = nullptr; return true; } @@ -647,7 +662,7 @@ void ReportRace(ThreadState *thr) { rep.AddLocation(addr_min, addr_max - addr_min); -#ifndef SANITIZER_GO +#if !SANITIZER_GO { // NOLINT Shadow s(thr->racy_state[1]); if (s.epoch() <= thr->last_sleep_clock.get(s.tid())) @@ -667,8 +682,16 @@ void PrintCurrentStack(ThreadState *thr, uptr pc) { PrintStack(SymbolizeStack(trace)); } +// Always inlining PrintCurrentStackSlow, because LocatePcInTrace assumes +// __sanitizer_print_stack_trace exists in the actual unwinded stack, but +// tail-call to PrintCurrentStackSlow breaks this assumption because +// __sanitizer_print_stack_trace disappears after tail-call. +// However, this solution is not reliable enough, please see dvyukov's comment +// http://reviews.llvm.org/D19148#406208 +// Also see PR27280 comment 2 and 3 for breaking examples and analysis. +ALWAYS_INLINE void PrintCurrentStackSlow(uptr pc) { -#ifndef SANITIZER_GO +#if !SANITIZER_GO BufferedStackTrace *ptrace = new(internal_alloc(MBlockStackTrace, sizeof(BufferedStackTrace))) BufferedStackTrace(); diff --git a/libsanitizer/tsan/tsan_rtl_thread.cc b/libsanitizer/tsan/tsan_rtl_thread.cc index 3939c77d41c3..6c4b74e2a76b 100644 --- a/libsanitizer/tsan/tsan_rtl_thread.cc +++ b/libsanitizer/tsan/tsan_rtl_thread.cc @@ -28,7 +28,7 @@ ThreadContext::ThreadContext(int tid) , epoch1() { } -#ifndef SANITIZER_GO +#if !SANITIZER_GO ThreadContext::~ThreadContext() { } #endif @@ -40,7 +40,7 @@ void ThreadContext::OnDead() { void ThreadContext::OnJoined(void *arg) { ThreadState *caller_thr = static_cast<ThreadState *>(arg); AcquireImpl(caller_thr, 0, &sync); - sync.Reset(&caller_thr->clock_cache); + sync.Reset(&caller_thr->proc()->clock_cache); } struct OnCreatedArgs { @@ -66,13 +66,13 @@ void ThreadContext::OnCreated(void *arg) { void ThreadContext::OnReset() { CHECK_EQ(sync.size(), 0); - FlushUnneededShadowMemory(GetThreadTrace(tid), TraceSize() * sizeof(Event)); - //!!! FlushUnneededShadowMemory(GetThreadTraceHeader(tid), sizeof(Trace)); + ReleaseMemoryToOS(GetThreadTrace(tid), TraceSize() * sizeof(Event)); + //!!! ReleaseMemoryToOS(GetThreadTraceHeader(tid), sizeof(Trace)); } void ThreadContext::OnDetached(void *arg) { ThreadState *thr1 = static_cast<ThreadState*>(arg); - sync.Reset(&thr1->clock_cache); + sync.Reset(&thr1->proc()->clock_cache); } struct OnStartedArgs { @@ -92,7 +92,7 @@ void ThreadContext::OnStarted(void *arg) { epoch1 = (u64)-1; new(thr) ThreadState(ctx, tid, unique_id, epoch0, reuse_count, args->stk_addr, args->stk_size, args->tls_addr, args->tls_size); -#ifndef SANITIZER_GO +#if !SANITIZER_GO thr->shadow_stack = &ThreadTrace(thr->tid)->shadow_stack[0]; thr->shadow_stack_pos = thr->shadow_stack; thr->shadow_stack_end = thr->shadow_stack + kShadowStackSize; @@ -104,13 +104,8 @@ void ThreadContext::OnStarted(void *arg) { thr->shadow_stack_pos = thr->shadow_stack; thr->shadow_stack_end = thr->shadow_stack + kInitStackSize; #endif -#ifndef SANITIZER_GO - AllocatorThreadStart(thr); -#endif - if (common_flags()->detect_deadlocks) { - thr->dd_pt = ctx->dd->CreatePhysicalThread(); + if (common_flags()->detect_deadlocks) thr->dd_lt = ctx->dd->CreateLogicalThread(unique_id); - } thr->fast_state.SetHistorySize(flags()->history_size); // Commit switch to the new part of the trace. // TraceAddEvent will reset stack0/mset0 in the new part for us. @@ -119,7 +114,7 @@ void ThreadContext::OnStarted(void *arg) { thr->fast_synch_epoch = epoch0; AcquireImpl(thr, 0, &sync); StatInc(thr, StatSyncAcquire); - sync.Reset(&thr->clock_cache); + sync.Reset(&thr->proc()->clock_cache); thr->is_inited = true; DPrintf("#%d: ThreadStart epoch=%zu stk_addr=%zx stk_size=%zx " "tls_addr=%zx tls_size=%zx\n", @@ -128,6 +123,12 @@ void ThreadContext::OnStarted(void *arg) { } void ThreadContext::OnFinished() { +#if SANITIZER_GO + internal_free(thr->shadow_stack); + thr->shadow_stack = nullptr; + thr->shadow_stack_pos = nullptr; + thr->shadow_stack_end = nullptr; +#endif if (!detached) { thr->fast_state.IncrementEpoch(); // Can't increment epoch w/o writing to the trace as well. @@ -136,15 +137,8 @@ void ThreadContext::OnFinished() { } epoch1 = thr->fast_state.epoch(); - if (common_flags()->detect_deadlocks) { - ctx->dd->DestroyPhysicalThread(thr->dd_pt); + if (common_flags()->detect_deadlocks) ctx->dd->DestroyLogicalThread(thr->dd_lt); - } - ctx->clock_alloc.FlushCache(&thr->clock_cache); - ctx->metamap.OnThreadIdle(thr); -#ifndef SANITIZER_GO - AllocatorThreadFinish(thr); -#endif thr->~ThreadState(); #if TSAN_COLLECT_STATS StatAggregate(ctx->stat, thr->stat); @@ -152,7 +146,7 @@ void ThreadContext::OnFinished() { thr = 0; } -#ifndef SANITIZER_GO +#if !SANITIZER_GO struct ThreadLeak { ThreadContext *tctx; int count; @@ -174,7 +168,7 @@ static void MaybeReportThreadLeak(ThreadContextBase *tctx_base, void *arg) { } #endif -#ifndef SANITIZER_GO +#if !SANITIZER_GO static void ReportIgnoresEnabled(ThreadContext *tctx, IgnoreSet *set) { if (tctx->tid == 0) { Printf("ThreadSanitizer: main thread finished with ignores enabled\n"); @@ -206,7 +200,7 @@ static void ThreadCheckIgnore(ThreadState *thr) {} void ThreadFinalize(ThreadState *thr) { ThreadCheckIgnore(thr); -#ifndef SANITIZER_GO +#if !SANITIZER_GO if (!flags()->report_thread_leaks) return; ThreadRegistryLock l(ctx->thread_registry); @@ -244,7 +238,7 @@ void ThreadStart(ThreadState *thr, int tid, uptr os_id) { uptr stk_size = 0; uptr tls_addr = 0; uptr tls_size = 0; -#ifndef SANITIZER_GO +#if !SANITIZER_GO GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size); if (tid) { @@ -275,7 +269,7 @@ void ThreadStart(ThreadState *thr, int tid, uptr os_id) { thr->tctx = (ThreadContext*)tr->GetThreadLocked(tid); tr->Unlock(); -#ifndef SANITIZER_GO +#if !SANITIZER_GO if (ctx->after_multithreaded_fork) { thr->ignore_interceptors++; ThreadIgnoreBegin(thr, 0); diff --git a/libsanitizer/tsan/tsan_stat.cc b/libsanitizer/tsan/tsan_stat.cc index 5eccca69312f..740cb86c69e3 100644 --- a/libsanitizer/tsan/tsan_stat.cc +++ b/libsanitizer/tsan/tsan_stat.cc @@ -166,6 +166,7 @@ void StatOutput(u64 *stat) { name[StatMtxFired] = " FiredSuppressions "; name[StatMtxRacy] = " RacyStacks "; name[StatMtxFD] = " FD "; + name[StatMtxGlobalProc] = " GlobalProc "; Printf("Statistics:\n"); for (int i = 0; i < StatCnt; i++) diff --git a/libsanitizer/tsan/tsan_stat.h b/libsanitizer/tsan/tsan_stat.h index 002570f42f12..788adeb56eaf 100644 --- a/libsanitizer/tsan/tsan_stat.h +++ b/libsanitizer/tsan/tsan_stat.h @@ -171,6 +171,7 @@ enum StatType { StatMtxFired, StatMtxRacy, StatMtxFD, + StatMtxGlobalProc, // This must be the last. StatCnt diff --git a/libsanitizer/tsan/tsan_suppressions.cc b/libsanitizer/tsan/tsan_suppressions.cc index 5c8d03dddd24..dc862b1b9adc 100644 --- a/libsanitizer/tsan/tsan_suppressions.cc +++ b/libsanitizer/tsan/tsan_suppressions.cc @@ -19,7 +19,7 @@ #include "tsan_mman.h" #include "tsan_platform.h" -#ifndef SANITIZER_GO +#if !SANITIZER_GO // Suppressions for true/false positives in standard libraries. static const char *const std_suppressions = // Libstdc++ 4.4 has data races in std::string. @@ -32,7 +32,8 @@ static const char *const std_suppressions = "race:std::_Sp_counted_ptr_inplace<std::thread::_Impl\n"; // Can be overriden in frontend. -extern "C" const char *WEAK __tsan_default_suppressions() { +SANITIZER_WEAK_DEFAULT_IMPL +const char *__tsan_default_suppressions() { return 0; } #endif @@ -51,7 +52,7 @@ void InitializeSuppressions() { suppression_ctx = new (suppression_placeholder) // NOLINT SuppressionContext(kSuppressionTypes, ARRAY_SIZE(kSuppressionTypes)); suppression_ctx->ParseFromFile(flags()->suppressions); -#ifndef SANITIZER_GO +#if !SANITIZER_GO suppression_ctx->Parse(__tsan_default_suppressions()); suppression_ctx->Parse(std_suppressions); #endif @@ -77,6 +78,8 @@ static const char *conv(ReportType typ) { return kSuppressionMutex; else if (typ == ReportTypeMutexDoubleLock) return kSuppressionMutex; + else if (typ == ReportTypeMutexInvalidAccess) + return kSuppressionMutex; else if (typ == ReportTypeMutexBadUnlock) return kSuppressionMutex; else if (typ == ReportTypeMutexBadReadLock) @@ -89,7 +92,7 @@ static const char *conv(ReportType typ) { return kSuppressionNone; else if (typ == ReportTypeDeadlock) return kSuppressionDeadlock; - Printf("ThreadSanitizer: unknown report type %d\n", typ), + Printf("ThreadSanitizer: unknown report type %d\n", typ); Die(); } @@ -156,8 +159,8 @@ void PrintMatchedSuppressions() { Printf("ThreadSanitizer: Matched %d suppressions (pid=%d):\n", hit_count, (int)internal_getpid()); for (uptr i = 0; i < matched.size(); i++) { - Printf("%d %s:%s\n", matched[i]->hit_count, matched[i]->type, - matched[i]->templ); + Printf("%d %s:%s\n", atomic_load_relaxed(&matched[i]->hit_count), + matched[i]->type, matched[i]->templ); } } } // namespace __tsan diff --git a/libsanitizer/tsan/tsan_symbolize.cc b/libsanitizer/tsan/tsan_symbolize.cc index 0e54562e385f..7b04782d1ad7 100644 --- a/libsanitizer/tsan/tsan_symbolize.cc +++ b/libsanitizer/tsan/tsan_symbolize.cc @@ -36,10 +36,10 @@ void ExitSymbolizer() { // May be overriden by JIT/JAVA/etc, // whatever produces PCs marked with kExternalPCBit. -extern "C" bool WEAK __tsan_symbolize_external(uptr pc, - char *func_buf, uptr func_siz, - char *file_buf, uptr file_siz, - int *line, int *col) { +SANITIZER_WEAK_DEFAULT_IMPL +bool __tsan_symbolize_external(uptr pc, char *func_buf, uptr func_siz, + char *file_buf, uptr file_siz, int *line, + int *col) { return false; } @@ -69,7 +69,7 @@ ReportLocation *SymbolizeData(uptr addr) { if (!Symbolizer::GetOrInit()->SymbolizeData(addr, &info)) return 0; ReportLocation *ent = ReportLocation::New(ReportLocationGlobal); - ent->global = info; + internal_memcpy(&ent->global, &info, sizeof(info)); return ent; } diff --git a/libsanitizer/tsan/tsan_sync.cc b/libsanitizer/tsan/tsan_sync.cc index 91ad8c8b2284..0ee2295e69e9 100644 --- a/libsanitizer/tsan/tsan_sync.cc +++ b/libsanitizer/tsan/tsan_sync.cc @@ -28,13 +28,13 @@ void SyncVar::Init(ThreadState *thr, uptr pc, uptr addr, u64 uid) { this->next = 0; creation_stack_id = 0; - if (kCppMode) // Go does not use them + if (!SANITIZER_GO) // Go does not use them creation_stack_id = CurrentStackId(thr, pc); if (common_flags()->detect_deadlocks) DDMutexInit(thr, pc, this); } -void SyncVar::Reset(ThreadState *thr) { +void SyncVar::Reset(Processor *proc) { uid = 0; creation_stack_id = 0; owner_tid = kInvalidTid; @@ -45,12 +45,12 @@ void SyncVar::Reset(ThreadState *thr) { is_broken = 0; is_linker_init = 0; - if (thr == 0) { + if (proc == 0) { CHECK_EQ(clock.size(), 0); CHECK_EQ(read_clock.size(), 0); } else { - clock.Reset(&thr->clock_cache); - read_clock.Reset(&thr->clock_cache); + clock.Reset(&proc->clock_cache); + read_clock.Reset(&proc->clock_cache); } } @@ -59,7 +59,7 @@ MetaMap::MetaMap() { } void MetaMap::AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz) { - u32 idx = block_alloc_.Alloc(&thr->block_cache); + u32 idx = block_alloc_.Alloc(&thr->proc()->block_cache); MBlock *b = block_alloc_.Map(idx); b->siz = sz; b->tid = thr->tid; @@ -69,16 +69,16 @@ void MetaMap::AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz) { *meta = idx | kFlagBlock; } -uptr MetaMap::FreeBlock(ThreadState *thr, uptr pc, uptr p) { +uptr MetaMap::FreeBlock(Processor *proc, uptr p) { MBlock* b = GetBlock(p); if (b == 0) return 0; uptr sz = RoundUpTo(b->siz, kMetaShadowCell); - FreeRange(thr, pc, p, sz); + FreeRange(proc, p, sz); return sz; } -bool MetaMap::FreeRange(ThreadState *thr, uptr pc, uptr p, uptr sz) { +bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz) { bool has_something = false; u32 *meta = MemToMeta(p); u32 *end = MemToMeta(p + sz); @@ -94,14 +94,14 @@ bool MetaMap::FreeRange(ThreadState *thr, uptr pc, uptr p, uptr sz) { has_something = true; while (idx != 0) { if (idx & kFlagBlock) { - block_alloc_.Free(&thr->block_cache, idx & ~kFlagMask); + block_alloc_.Free(&proc->block_cache, idx & ~kFlagMask); break; } else if (idx & kFlagSync) { DCHECK(idx & kFlagSync); SyncVar *s = sync_alloc_.Map(idx & ~kFlagMask); u32 next = s->next; - s->Reset(thr); - sync_alloc_.Free(&thr->sync_cache, idx & ~kFlagMask); + s->Reset(proc); + sync_alloc_.Free(&proc->sync_cache, idx & ~kFlagMask); idx = next; } else { CHECK(0); @@ -117,24 +117,30 @@ bool MetaMap::FreeRange(ThreadState *thr, uptr pc, uptr p, uptr sz) { // which can be huge. The function probes pages one-by-one until it finds a page // without meta objects, at this point it stops freeing meta objects. Because // thread stacks grow top-down, we do the same starting from end as well. -void MetaMap::ResetRange(ThreadState *thr, uptr pc, uptr p, uptr sz) { +void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) { + if (SANITIZER_GO) { + // UnmapOrDie/MmapFixedNoReserve does not work on Windows, + // so we do the optimization only for C/C++. + FreeRange(proc, p, sz); + return; + } const uptr kMetaRatio = kMetaShadowCell / kMetaShadowSize; const uptr kPageSize = GetPageSizeCached() * kMetaRatio; if (sz <= 4 * kPageSize) { // If the range is small, just do the normal free procedure. - FreeRange(thr, pc, p, sz); + FreeRange(proc, p, sz); return; } // First, round both ends of the range to page size. uptr diff = RoundUp(p, kPageSize) - p; if (diff != 0) { - FreeRange(thr, pc, p, diff); + FreeRange(proc, p, diff); p += diff; sz -= diff; } diff = p + sz - RoundDown(p + sz, kPageSize); if (diff != 0) { - FreeRange(thr, pc, p + sz - diff, diff); + FreeRange(proc, p + sz - diff, diff); sz -= diff; } // Now we must have a non-empty page-aligned range. @@ -144,18 +150,21 @@ void MetaMap::ResetRange(ThreadState *thr, uptr pc, uptr p, uptr sz) { const uptr p0 = p; const uptr sz0 = sz; // Probe start of the range. - while (sz > 0) { - bool has_something = FreeRange(thr, pc, p, kPageSize); + for (uptr checked = 0; sz > 0; checked += kPageSize) { + bool has_something = FreeRange(proc, p, kPageSize); p += kPageSize; sz -= kPageSize; - if (!has_something) + if (!has_something && checked > (128 << 10)) break; } // Probe end of the range. - while (sz > 0) { - bool has_something = FreeRange(thr, pc, p - kPageSize, kPageSize); + for (uptr checked = 0; sz > 0; checked += kPageSize) { + bool has_something = FreeRange(proc, p + sz - kPageSize, kPageSize); sz -= kPageSize; - if (!has_something) + // Stacks grow down, so sync object are most likely at the end of the region + // (if it is a stack). The very end of the stack is TLS and tsan increases + // TLS by at least 256K, so check at least 512K. + if (!has_something && checked > (512 << 10)) break; } // Finally, page out the whole range (including the parts that we've just @@ -187,8 +196,8 @@ SyncVar* MetaMap::GetOrCreateAndLock(ThreadState *thr, uptr pc, return GetAndLock(thr, pc, addr, write_lock, true); } -SyncVar* MetaMap::GetIfExistsAndLock(uptr addr) { - return GetAndLock(0, 0, addr, true, false); +SyncVar* MetaMap::GetIfExistsAndLock(uptr addr, bool write_lock) { + return GetAndLock(0, 0, addr, write_lock, false); } SyncVar* MetaMap::GetAndLock(ThreadState *thr, uptr pc, @@ -208,8 +217,8 @@ SyncVar* MetaMap::GetAndLock(ThreadState *thr, uptr pc, SyncVar * s = sync_alloc_.Map(idx & ~kFlagMask); if (s->addr == addr) { if (myidx != 0) { - mys->Reset(thr); - sync_alloc_.Free(&thr->sync_cache, myidx); + mys->Reset(thr->proc()); + sync_alloc_.Free(&thr->proc()->sync_cache, myidx); } if (write_lock) s->mtx.Lock(); @@ -228,7 +237,7 @@ SyncVar* MetaMap::GetAndLock(ThreadState *thr, uptr pc, if (myidx == 0) { const u64 uid = atomic_fetch_add(&uid_gen_, 1, memory_order_relaxed); - myidx = sync_alloc_.Alloc(&thr->sync_cache); + myidx = sync_alloc_.Alloc(&thr->proc()->sync_cache); mys = sync_alloc_.Map(myidx); mys->Init(thr, pc, addr, uid); } @@ -277,9 +286,9 @@ void MetaMap::MoveMemory(uptr src, uptr dst, uptr sz) { } } -void MetaMap::OnThreadIdle(ThreadState *thr) { - block_alloc_.FlushCache(&thr->block_cache); - sync_alloc_.FlushCache(&thr->sync_cache); +void MetaMap::OnProcIdle(Processor *proc) { + block_alloc_.FlushCache(&proc->block_cache); + sync_alloc_.FlushCache(&proc->sync_cache); } } // namespace __tsan diff --git a/libsanitizer/tsan/tsan_sync.h b/libsanitizer/tsan/tsan_sync.h index 50bc872275de..48d7027b428e 100644 --- a/libsanitizer/tsan/tsan_sync.h +++ b/libsanitizer/tsan/tsan_sync.h @@ -45,19 +45,19 @@ struct SyncVar { SyncClock clock; void Init(ThreadState *thr, uptr pc, uptr addr, u64 uid); - void Reset(ThreadState *thr); + void Reset(Processor *proc); u64 GetId() const { - // 47 lsb is addr, then 14 bits is low part of uid, then 3 zero bits. - return GetLsb((u64)addr | (uid << 47), 61); + // 48 lsb is addr, then 14 bits is low part of uid, then 2 zero bits. + return GetLsb((u64)addr | (uid << 48), 60); } bool CheckId(u64 uid) const { CHECK_EQ(uid, GetLsb(uid, 14)); return GetLsb(this->uid, 14) == uid; } static uptr SplitId(u64 id, u64 *uid) { - *uid = id >> 47; - return (uptr)GetLsb(id, 47); + *uid = id >> 48; + return (uptr)GetLsb(id, 48); } }; @@ -70,18 +70,18 @@ class MetaMap { MetaMap(); void AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz); - uptr FreeBlock(ThreadState *thr, uptr pc, uptr p); - bool FreeRange(ThreadState *thr, uptr pc, uptr p, uptr sz); - void ResetRange(ThreadState *thr, uptr pc, uptr p, uptr sz); + uptr FreeBlock(Processor *proc, uptr p); + bool FreeRange(Processor *proc, uptr p, uptr sz); + void ResetRange(Processor *proc, uptr p, uptr sz); MBlock* GetBlock(uptr p); SyncVar* GetOrCreateAndLock(ThreadState *thr, uptr pc, uptr addr, bool write_lock); - SyncVar* GetIfExistsAndLock(uptr addr); + SyncVar* GetIfExistsAndLock(uptr addr, bool write_lock); void MoveMemory(uptr src, uptr dst, uptr sz); - void OnThreadIdle(ThreadState *thr); + void OnProcIdle(Processor *proc); private: static const u32 kFlagMask = 3u << 30; diff --git a/libsanitizer/tsan/tsan_trace.h b/libsanitizer/tsan/tsan_trace.h index a27efa9fd7a9..1ea733bfd07a 100644 --- a/libsanitizer/tsan/tsan_trace.h +++ b/libsanitizer/tsan/tsan_trace.h @@ -40,7 +40,7 @@ enum EventType { typedef u64 Event; struct TraceHeader { -#ifndef SANITIZER_GO +#if !SANITIZER_GO BufferedStackTrace stack0; // Start stack for the trace. #else VarSizeStackTrace stack0; @@ -53,7 +53,7 @@ struct TraceHeader { struct Trace { Mutex mtx; -#ifndef SANITIZER_GO +#if !SANITIZER_GO // Must be last to catch overflow as paging fault. // Go shadow stack is dynamically allocated. uptr shadow_stack[kShadowStackSize]; |