diff options
Diffstat (limited to 'libsanitizer/sanitizer_common')
77 files changed, 5643 insertions, 2332 deletions
diff --git a/libsanitizer/sanitizer_common/Makefile.am b/libsanitizer/sanitizer_common/Makefile.am index ee7a3f1dd2c5..8b6336fd3ade 100644 --- a/libsanitizer/sanitizer_common/Makefile.am +++ b/libsanitizer/sanitizer_common/Makefile.am @@ -32,6 +32,7 @@ sanitizer_common_files = \ sanitizer_libignore.cc \ sanitizer_linux.cc \ sanitizer_linux_libcdep.cc \ + sanitizer_linux_s390.cc \ sanitizer_mac.cc \ sanitizer_persistent_allocator.cc \ sanitizer_platform_limits_linux.cc \ @@ -55,6 +56,7 @@ sanitizer_common_files = \ sanitizer_symbolizer_libcdep.cc \ sanitizer_symbolizer_posix_libcdep.cc \ sanitizer_symbolizer_win.cc \ + sanitizer_termination.cc \ sanitizer_thread_registry.cc \ sanitizer_tls_get_addr.cc \ sanitizer_unwind_linux_libcdep.cc \ @@ -62,6 +64,9 @@ sanitizer_common_files = \ libsanitizer_common_la_SOURCES = $(sanitizer_common_files) +EXTRA_libsanitizer_common_la_SOURCES = sanitizer_linux_mips64.S sanitizer_linux_x86_64.S +libsanitizer_common_la_LIBADD = $(SANITIZER_COMMON_TARGET_DEPENDENT_OBJECTS) +libsanitizer_common_la_DEPENDENCIES = $(SANITIZER_COMMON_TARGET_DEPENDENT_OBJECTS) # Work around what appears to be a GNU make bug handling MAKEFLAGS # values defined in terms of make variables, as is the case for CC and diff --git a/libsanitizer/sanitizer_common/Makefile.in b/libsanitizer/sanitizer_common/Makefile.in index 4b008ad7ae62..052802e176a6 100644 --- a/libsanitizer/sanitizer_common/Makefile.in +++ b/libsanitizer/sanitizer_common/Makefile.in @@ -79,7 +79,7 @@ CONFIG_HEADER = $(top_builddir)/config.h CONFIG_CLEAN_FILES = CONFIG_CLEAN_VPATH_FILES = LTLIBRARIES = $(noinst_LTLIBRARIES) -libsanitizer_common_la_LIBADD = +am__DEPENDENCIES_1 = am__objects_1 = sanitizer_allocator.lo sanitizer_common.lo \ sanitizer_common_libcdep.lo sanitizer_coverage_libcdep.lo \ sanitizer_coverage_mapping_libcdep.lo \ @@ -87,8 +87,8 @@ am__objects_1 = sanitizer_allocator.lo sanitizer_common.lo \ sanitizer_deadlock_detector2.lo sanitizer_flags.lo \ sanitizer_flag_parser.lo sanitizer_libc.lo \ sanitizer_libignore.lo sanitizer_linux.lo \ - sanitizer_linux_libcdep.lo sanitizer_mac.lo \ - sanitizer_persistent_allocator.lo \ + sanitizer_linux_libcdep.lo sanitizer_linux_s390.lo \ + sanitizer_mac.lo sanitizer_persistent_allocator.lo \ sanitizer_platform_limits_linux.lo \ sanitizer_platform_limits_posix.lo sanitizer_posix.lo \ sanitizer_posix_libcdep.lo sanitizer_printf.lo \ @@ -102,15 +102,20 @@ am__objects_1 = sanitizer_allocator.lo sanitizer_common.lo \ sanitizer_symbolizer_libbacktrace.lo \ sanitizer_symbolizer_libcdep.lo \ sanitizer_symbolizer_posix_libcdep.lo \ - sanitizer_symbolizer_win.lo sanitizer_thread_registry.lo \ - sanitizer_tls_get_addr.lo sanitizer_unwind_linux_libcdep.lo \ - sanitizer_win.lo + sanitizer_symbolizer_win.lo sanitizer_termination.lo \ + sanitizer_thread_registry.lo sanitizer_tls_get_addr.lo \ + sanitizer_unwind_linux_libcdep.lo sanitizer_win.lo am_libsanitizer_common_la_OBJECTS = $(am__objects_1) libsanitizer_common_la_OBJECTS = $(am_libsanitizer_common_la_OBJECTS) DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) depcomp = $(SHELL) $(top_srcdir)/../depcomp am__depfiles_maybe = depfiles am__mv = mv -f +CPPASCOMPILE = $(CCAS) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CCASFLAGS) $(CCASFLAGS) +LTCPPASCOMPILE = $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ + --mode=compile $(CCAS) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CCASFLAGS) $(CCASFLAGS) CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) LTCXXCOMPILE = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ @@ -120,7 +125,17 @@ CXXLD = $(CXX) CXXLINK = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ --mode=link $(CXXLD) $(AM_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ $(LDFLAGS) -o $@ -SOURCES = $(libsanitizer_common_la_SOURCES) +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ + --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +CCLD = $(CC) +LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ + --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +SOURCES = $(libsanitizer_common_la_SOURCES) \ + $(EXTRA_libsanitizer_common_la_SOURCES) am__can_run_installinfo = \ case $$AM_UPDATE_INFO_DIR in \ n|no|NO) false;; \ @@ -198,6 +213,7 @@ PACKAGE_VERSION = @PACKAGE_VERSION@ PATH_SEPARATOR = @PATH_SEPARATOR@ RANLIB = @RANLIB@ RPC_DEFS = @RPC_DEFS@ +SANITIZER_COMMON_TARGET_DEPENDENT_OBJECTS = @SANITIZER_COMMON_TARGET_DEPENDENT_OBJECTS@ SED = @SED@ SET_MAKE = @SET_MAKE@ SHELL = @SHELL@ @@ -296,6 +312,7 @@ sanitizer_common_files = \ sanitizer_libignore.cc \ sanitizer_linux.cc \ sanitizer_linux_libcdep.cc \ + sanitizer_linux_s390.cc \ sanitizer_mac.cc \ sanitizer_persistent_allocator.cc \ sanitizer_platform_limits_linux.cc \ @@ -319,12 +336,16 @@ sanitizer_common_files = \ sanitizer_symbolizer_libcdep.cc \ sanitizer_symbolizer_posix_libcdep.cc \ sanitizer_symbolizer_win.cc \ + sanitizer_termination.cc \ sanitizer_thread_registry.cc \ sanitizer_tls_get_addr.cc \ sanitizer_unwind_linux_libcdep.cc \ sanitizer_win.cc libsanitizer_common_la_SOURCES = $(sanitizer_common_files) +EXTRA_libsanitizer_common_la_SOURCES = sanitizer_linux_mips64.S sanitizer_linux_x86_64.S +libsanitizer_common_la_LIBADD = $(SANITIZER_COMMON_TARGET_DEPENDENT_OBJECTS) +libsanitizer_common_la_DEPENDENCIES = $(SANITIZER_COMMON_TARGET_DEPENDENT_OBJECTS) # Work around what appears to be a GNU make bug handling MAKEFLAGS # values defined in terms of make variables, as is the case for CC and @@ -368,7 +389,7 @@ MAKEOVERRIDES = all: all-am .SUFFIXES: -.SUFFIXES: .cc .lo .o .obj +.SUFFIXES: .S .cc .lo .o .obj $(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) @for dep in $?; do \ case '$(am__configure_deps)' in \ @@ -430,6 +451,9 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_libignore.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_linux.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_linux_libcdep.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_linux_mips64.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_linux_s390.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_linux_x86_64.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_mac.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_persistent_allocator.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_platform_limits_linux.Plo@am__quote@ @@ -453,11 +477,33 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_symbolizer_mac.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_symbolizer_posix_libcdep.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_symbolizer_win.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_termination.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_thread_registry.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_tls_get_addr.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_unwind_linux_libcdep.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_win.Plo@am__quote@ +.S.o: +@am__fastdepCCAS_TRUE@ $(CPPASCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCCAS_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCCAS_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCCAS_FALSE@ DEPDIR=$(DEPDIR) $(CCASDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCCAS_FALSE@ $(CPPASCOMPILE) -c -o $@ $< + +.S.obj: +@am__fastdepCCAS_TRUE@ $(CPPASCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCCAS_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCCAS_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCCAS_FALSE@ DEPDIR=$(DEPDIR) $(CCASDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCCAS_FALSE@ $(CPPASCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.S.lo: +@am__fastdepCCAS_TRUE@ $(LTCPPASCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCCAS_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCCAS_FALSE@ source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCCAS_FALSE@ DEPDIR=$(DEPDIR) $(CCASDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCCAS_FALSE@ $(LTCPPASCOMPILE) -c -o $@ $< + .cc.o: @am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< @am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator.cc b/libsanitizer/sanitizer_common/sanitizer_allocator.cc index b5e0bab91f15..2755853acbcd 100644 --- a/libsanitizer/sanitizer_common/sanitizer_allocator.cc +++ b/libsanitizer/sanitizer_common/sanitizer_allocator.cc @@ -11,39 +11,72 @@ //===----------------------------------------------------------------------===// #include "sanitizer_allocator.h" + #include "sanitizer_allocator_internal.h" +#include "sanitizer_atomic.h" #include "sanitizer_common.h" namespace __sanitizer { // ThreadSanitizer for Go uses libc malloc/free. -#if defined(SANITIZER_GO) || defined(SANITIZER_USE_MALLOC) +#if SANITIZER_GO || defined(SANITIZER_USE_MALLOC) # if SANITIZER_LINUX && !SANITIZER_ANDROID extern "C" void *__libc_malloc(uptr size); +# if !SANITIZER_GO +extern "C" void *__libc_memalign(uptr alignment, uptr size); +# endif +extern "C" void *__libc_realloc(void *ptr, uptr size); extern "C" void __libc_free(void *ptr); -# define LIBC_MALLOC __libc_malloc -# define LIBC_FREE __libc_free # else # include <stdlib.h> -# define LIBC_MALLOC malloc -# define LIBC_FREE free +# define __libc_malloc malloc +# if !SANITIZER_GO +static void *__libc_memalign(uptr alignment, uptr size) { + void *p; + uptr error = posix_memalign(&p, alignment, size); + if (error) return nullptr; + return p; +} +# endif +# define __libc_realloc realloc +# define __libc_free free # endif -static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache) { +static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache, + uptr alignment) { + (void)cache; +#if !SANITIZER_GO + if (alignment == 0) + return __libc_malloc(size); + else + return __libc_memalign(alignment, size); +#else + // Windows does not provide __libc_memalign/posix_memalign. It provides + // __aligned_malloc, but the allocated blocks can't be passed to free, + // they need to be passed to __aligned_free. InternalAlloc interface does + // not account for such requirement. Alignemnt does not seem to be used + // anywhere in runtime, so just call __libc_malloc for now. + DCHECK_EQ(alignment, 0); + return __libc_malloc(size); +#endif +} + +static void *RawInternalRealloc(void *ptr, uptr size, + InternalAllocatorCache *cache) { (void)cache; - return LIBC_MALLOC(size); + return __libc_realloc(ptr, size); } static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) { (void)cache; - LIBC_FREE(ptr); + __libc_free(ptr); } InternalAllocator *internal_allocator() { return 0; } -#else // SANITIZER_GO +#else // SANITIZER_GO || defined(SANITIZER_USE_MALLOC) static ALIGNED(64) char internal_alloc_placeholder[sizeof(InternalAllocator)]; static atomic_uint8_t internal_allocator_initialized; @@ -66,13 +99,26 @@ InternalAllocator *internal_allocator() { return internal_allocator_instance; } -static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache) { +static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache, + uptr alignment) { + if (alignment == 0) alignment = 8; + if (cache == 0) { + SpinMutexLock l(&internal_allocator_cache_mu); + return internal_allocator()->Allocate(&internal_allocator_cache, size, + alignment, false); + } + return internal_allocator()->Allocate(cache, size, alignment, false); +} + +static void *RawInternalRealloc(void *ptr, uptr size, + InternalAllocatorCache *cache) { + uptr alignment = 8; if (cache == 0) { SpinMutexLock l(&internal_allocator_cache_mu); - return internal_allocator()->Allocate(&internal_allocator_cache, size, 8, - false); + return internal_allocator()->Reallocate(&internal_allocator_cache, ptr, + size, alignment); } - return internal_allocator()->Allocate(cache, size, 8, false); + return internal_allocator()->Reallocate(cache, ptr, size, alignment); } static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) { @@ -83,20 +129,42 @@ static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) { internal_allocator()->Deallocate(cache, ptr); } -#endif // SANITIZER_GO +#endif // SANITIZER_GO || defined(SANITIZER_USE_MALLOC) const u64 kBlockMagic = 0x6A6CB03ABCEBC041ull; -void *InternalAlloc(uptr size, InternalAllocatorCache *cache) { +void *InternalAlloc(uptr size, InternalAllocatorCache *cache, uptr alignment) { if (size + sizeof(u64) < size) return nullptr; - void *p = RawInternalAlloc(size + sizeof(u64), cache); + void *p = RawInternalAlloc(size + sizeof(u64), cache, alignment); if (!p) return nullptr; ((u64*)p)[0] = kBlockMagic; return (char*)p + sizeof(u64); } +void *InternalRealloc(void *addr, uptr size, InternalAllocatorCache *cache) { + if (!addr) + return InternalAlloc(size, cache); + if (size + sizeof(u64) < size) + return nullptr; + addr = (char*)addr - sizeof(u64); + size = size + sizeof(u64); + CHECK_EQ(kBlockMagic, ((u64*)addr)[0]); + void *p = RawInternalRealloc(addr, size, cache); + if (!p) + return nullptr; + return (char*)p + sizeof(u64); +} + +void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache) { + if (CallocShouldReturnNullDueToOverflow(count, size)) + return internal_allocator()->ReturnNullOrDieOnBadRequest(); + void *p = InternalAlloc(count * size, cache); + if (p) internal_memset(p, 0, count * size); + return p; +} + void InternalFree(void *addr, InternalAllocatorCache *cache) { if (!addr) return; @@ -138,7 +206,12 @@ bool CallocShouldReturnNullDueToOverflow(uptr size, uptr n) { return (max / size) < n; } -void NORETURN ReportAllocatorCannotReturnNull() { +static atomic_uint8_t reporting_out_of_memory = {0}; + +bool IsReportingOOM() { return atomic_load_relaxed(&reporting_out_of_memory); } + +void NORETURN ReportAllocatorCannotReturnNull(bool out_of_memory) { + if (out_of_memory) atomic_store_relaxed(&reporting_out_of_memory, 1); Report("%s's allocator is terminating the process instead of returning 0\n", SanitizerToolName); Report("If you don't like this behavior set allocator_may_return_null=1\n"); diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator.h b/libsanitizer/sanitizer_common/sanitizer_allocator.h index d98528c722ca..ba50acddbbf3 100644 --- a/libsanitizer/sanitizer_common/sanitizer_allocator.h +++ b/libsanitizer/sanitizer_common/sanitizer_allocator.h @@ -18,271 +18,16 @@ #include "sanitizer_list.h" #include "sanitizer_mutex.h" #include "sanitizer_lfstack.h" +#include "sanitizer_procmaps.h" namespace __sanitizer { -// Prints error message and kills the program. -void NORETURN ReportAllocatorCannotReturnNull(); - -// SizeClassMap maps allocation sizes into size classes and back. -// Class 0 corresponds to size 0. -// Classes 1 - 16 correspond to sizes 16 to 256 (size = class_id * 16). -// Next 4 classes: 256 + i * 64 (i = 1 to 4). -// Next 4 classes: 512 + i * 128 (i = 1 to 4). -// ... -// Next 4 classes: 2^k + i * 2^(k-2) (i = 1 to 4). -// Last class corresponds to kMaxSize = 1 << kMaxSizeLog. -// -// This structure of the size class map gives us: -// - Efficient table-free class-to-size and size-to-class functions. -// - Difference between two consequent size classes is betweed 14% and 25% -// -// This class also gives a hint to a thread-caching allocator about the amount -// of chunks that need to be cached per-thread: -// - kMaxNumCached is the maximal number of chunks per size class. -// - (1 << kMaxBytesCachedLog) is the maximal number of bytes per size class. -// -// Part of output of SizeClassMap::Print(): -// c00 => s: 0 diff: +0 00% l 0 cached: 0 0; id 0 -// c01 => s: 16 diff: +16 00% l 4 cached: 256 4096; id 1 -// c02 => s: 32 diff: +16 100% l 5 cached: 256 8192; id 2 -// c03 => s: 48 diff: +16 50% l 5 cached: 256 12288; id 3 -// c04 => s: 64 diff: +16 33% l 6 cached: 256 16384; id 4 -// c05 => s: 80 diff: +16 25% l 6 cached: 256 20480; id 5 -// c06 => s: 96 diff: +16 20% l 6 cached: 256 24576; id 6 -// c07 => s: 112 diff: +16 16% l 6 cached: 256 28672; id 7 -// -// c08 => s: 128 diff: +16 14% l 7 cached: 256 32768; id 8 -// c09 => s: 144 diff: +16 12% l 7 cached: 256 36864; id 9 -// c10 => s: 160 diff: +16 11% l 7 cached: 256 40960; id 10 -// c11 => s: 176 diff: +16 10% l 7 cached: 256 45056; id 11 -// c12 => s: 192 diff: +16 09% l 7 cached: 256 49152; id 12 -// c13 => s: 208 diff: +16 08% l 7 cached: 256 53248; id 13 -// c14 => s: 224 diff: +16 07% l 7 cached: 256 57344; id 14 -// c15 => s: 240 diff: +16 07% l 7 cached: 256 61440; id 15 -// -// c16 => s: 256 diff: +16 06% l 8 cached: 256 65536; id 16 -// c17 => s: 320 diff: +64 25% l 8 cached: 204 65280; id 17 -// c18 => s: 384 diff: +64 20% l 8 cached: 170 65280; id 18 -// c19 => s: 448 diff: +64 16% l 8 cached: 146 65408; id 19 -// -// c20 => s: 512 diff: +64 14% l 9 cached: 128 65536; id 20 -// c21 => s: 640 diff: +128 25% l 9 cached: 102 65280; id 21 -// c22 => s: 768 diff: +128 20% l 9 cached: 85 65280; id 22 -// c23 => s: 896 diff: +128 16% l 9 cached: 73 65408; id 23 -// -// c24 => s: 1024 diff: +128 14% l 10 cached: 64 65536; id 24 -// c25 => s: 1280 diff: +256 25% l 10 cached: 51 65280; id 25 -// c26 => s: 1536 diff: +256 20% l 10 cached: 42 64512; id 26 -// c27 => s: 1792 diff: +256 16% l 10 cached: 36 64512; id 27 -// -// ... -// -// c48 => s: 65536 diff: +8192 14% l 16 cached: 1 65536; id 48 -// c49 => s: 81920 diff: +16384 25% l 16 cached: 1 81920; id 49 -// c50 => s: 98304 diff: +16384 20% l 16 cached: 1 98304; id 50 -// c51 => s: 114688 diff: +16384 16% l 16 cached: 1 114688; id 51 -// -// c52 => s: 131072 diff: +16384 14% l 17 cached: 1 131072; id 52 - -template <uptr kMaxSizeLog, uptr kMaxNumCachedT, uptr kMaxBytesCachedLog> -class SizeClassMap { - static const uptr kMinSizeLog = 4; - static const uptr kMidSizeLog = kMinSizeLog + 4; - static const uptr kMinSize = 1 << kMinSizeLog; - static const uptr kMidSize = 1 << kMidSizeLog; - static const uptr kMidClass = kMidSize / kMinSize; - static const uptr S = 2; - static const uptr M = (1 << S) - 1; - - public: - static const uptr kMaxNumCached = kMaxNumCachedT; - // We transfer chunks between central and thread-local free lists in batches. - // For small size classes we allocate batches separately. - // For large size classes we use one of the chunks to store the batch. - struct TransferBatch { - TransferBatch *next; - uptr count; - void *batch[kMaxNumCached]; - }; - - static const uptr kMaxSize = 1UL << kMaxSizeLog; - static const uptr kNumClasses = - kMidClass + ((kMaxSizeLog - kMidSizeLog) << S) + 1; - COMPILER_CHECK(kNumClasses >= 32 && kNumClasses <= 256); - static const uptr kNumClassesRounded = - kNumClasses == 32 ? 32 : - kNumClasses <= 64 ? 64 : - kNumClasses <= 128 ? 128 : 256; - - static uptr Size(uptr class_id) { - if (class_id <= kMidClass) - return kMinSize * class_id; - class_id -= kMidClass; - uptr t = kMidSize << (class_id >> S); - return t + (t >> S) * (class_id & M); - } - - static uptr ClassID(uptr size) { - if (size <= kMidSize) - return (size + kMinSize - 1) >> kMinSizeLog; - if (size > kMaxSize) return 0; - uptr l = MostSignificantSetBitIndex(size); - uptr hbits = (size >> (l - S)) & M; - uptr lbits = size & ((1 << (l - S)) - 1); - uptr l1 = l - kMidSizeLog; - return kMidClass + (l1 << S) + hbits + (lbits > 0); - } - - static uptr MaxCached(uptr class_id) { - if (class_id == 0) return 0; - uptr n = (1UL << kMaxBytesCachedLog) / Size(class_id); - return Max<uptr>(1, Min(kMaxNumCached, n)); - } - - static void Print() { - uptr prev_s = 0; - uptr total_cached = 0; - for (uptr i = 0; i < kNumClasses; i++) { - uptr s = Size(i); - if (s >= kMidSize / 2 && (s & (s - 1)) == 0) - Printf("\n"); - uptr d = s - prev_s; - uptr p = prev_s ? (d * 100 / prev_s) : 0; - uptr l = s ? MostSignificantSetBitIndex(s) : 0; - uptr cached = MaxCached(i) * s; - Printf("c%02zd => s: %zd diff: +%zd %02zd%% l %zd " - "cached: %zd %zd; id %zd\n", - i, Size(i), d, p, l, MaxCached(i), cached, ClassID(s)); - total_cached += cached; - prev_s = s; - } - Printf("Total cached: %zd\n", total_cached); - } - - static bool SizeClassRequiresSeparateTransferBatch(uptr class_id) { - return Size(class_id) < sizeof(TransferBatch) - - sizeof(uptr) * (kMaxNumCached - MaxCached(class_id)); - } - - static void Validate() { - for (uptr c = 1; c < kNumClasses; c++) { - // Printf("Validate: c%zd\n", c); - uptr s = Size(c); - CHECK_NE(s, 0U); - CHECK_EQ(ClassID(s), c); - if (c != kNumClasses - 1) - CHECK_EQ(ClassID(s + 1), c + 1); - CHECK_EQ(ClassID(s - 1), c); - if (c) - CHECK_GT(Size(c), Size(c-1)); - } - CHECK_EQ(ClassID(kMaxSize + 1), 0); - - for (uptr s = 1; s <= kMaxSize; s++) { - uptr c = ClassID(s); - // Printf("s%zd => c%zd\n", s, c); - CHECK_LT(c, kNumClasses); - CHECK_GE(Size(c), s); - if (c > 0) - CHECK_LT(Size(c-1), s); - } - } -}; +// Returns true if ReportAllocatorCannotReturnNull(true) was called. +// Can be use to avoid memory hungry operations. +bool IsReportingOOM(); -typedef SizeClassMap<17, 128, 16> DefaultSizeClassMap; -typedef SizeClassMap<17, 64, 14> CompactSizeClassMap; -template<class SizeClassAllocator> struct SizeClassAllocatorLocalCache; - -// Memory allocator statistics -enum AllocatorStat { - AllocatorStatAllocated, - AllocatorStatMapped, - AllocatorStatCount -}; - -typedef uptr AllocatorStatCounters[AllocatorStatCount]; - -// Per-thread stats, live in per-thread cache. -class AllocatorStats { - public: - void Init() { - internal_memset(this, 0, sizeof(*this)); - } - void InitLinkerInitialized() {} - - void Add(AllocatorStat i, uptr v) { - v += atomic_load(&stats_[i], memory_order_relaxed); - atomic_store(&stats_[i], v, memory_order_relaxed); - } - - void Sub(AllocatorStat i, uptr v) { - v = atomic_load(&stats_[i], memory_order_relaxed) - v; - atomic_store(&stats_[i], v, memory_order_relaxed); - } - - void Set(AllocatorStat i, uptr v) { - atomic_store(&stats_[i], v, memory_order_relaxed); - } - - uptr Get(AllocatorStat i) const { - return atomic_load(&stats_[i], memory_order_relaxed); - } - - private: - friend class AllocatorGlobalStats; - AllocatorStats *next_; - AllocatorStats *prev_; - atomic_uintptr_t stats_[AllocatorStatCount]; -}; - -// Global stats, used for aggregation and querying. -class AllocatorGlobalStats : public AllocatorStats { - public: - void InitLinkerInitialized() { - next_ = this; - prev_ = this; - } - void Init() { - internal_memset(this, 0, sizeof(*this)); - InitLinkerInitialized(); - } - - void Register(AllocatorStats *s) { - SpinMutexLock l(&mu_); - s->next_ = next_; - s->prev_ = this; - next_->prev_ = s; - next_ = s; - } - - void Unregister(AllocatorStats *s) { - SpinMutexLock l(&mu_); - s->prev_->next_ = s->next_; - s->next_->prev_ = s->prev_; - for (int i = 0; i < AllocatorStatCount; i++) - Add(AllocatorStat(i), s->Get(AllocatorStat(i))); - } - - void Get(AllocatorStatCounters s) const { - internal_memset(s, 0, AllocatorStatCount * sizeof(uptr)); - SpinMutexLock l(&mu_); - const AllocatorStats *stats = this; - for (;;) { - for (int i = 0; i < AllocatorStatCount; i++) - s[i] += stats->Get(AllocatorStat(i)); - stats = stats->next_; - if (stats == this) - break; - } - // All stats must be non-negative. - for (int i = 0; i < AllocatorStatCount; i++) - s[i] = ((sptr)s[i]) >= 0 ? s[i] : 0; - } - - private: - mutable SpinMutex mu_; -}; +// Prints error message and kills the program. +void NORETURN ReportAllocatorCannotReturnNull(bool out_of_memory); // Allocators call these callbacks on mmap/munmap. struct NoOpMapUnmapCallback { @@ -293,1161 +38,18 @@ struct NoOpMapUnmapCallback { // Callback type for iterating over chunks. typedef void (*ForEachChunkCallback)(uptr chunk, void *arg); -// SizeClassAllocator64 -- allocator for 64-bit address space. -// -// Space: a portion of address space of kSpaceSize bytes starting at -// a fixed address (kSpaceBeg). Both constants are powers of two and -// kSpaceBeg is kSpaceSize-aligned. -// At the beginning the entire space is mprotect-ed, then small parts of it -// are mapped on demand. -// -// Region: a part of Space dedicated to a single size class. -// There are kNumClasses Regions of equal size. -// -// UserChunk: a piece of memory returned to user. -// MetaChunk: kMetadataSize bytes of metadata associated with a UserChunk. -// -// A Region looks like this: -// UserChunk1 ... UserChunkN <gap> MetaChunkN ... MetaChunk1 -template <const uptr kSpaceBeg, const uptr kSpaceSize, - const uptr kMetadataSize, class SizeClassMap, - class MapUnmapCallback = NoOpMapUnmapCallback> -class SizeClassAllocator64 { - public: - typedef typename SizeClassMap::TransferBatch Batch; - typedef SizeClassAllocator64<kSpaceBeg, kSpaceSize, kMetadataSize, - SizeClassMap, MapUnmapCallback> ThisT; - typedef SizeClassAllocatorLocalCache<ThisT> AllocatorCache; - - void Init() { - CHECK_EQ(kSpaceBeg, - reinterpret_cast<uptr>(MmapNoAccess(kSpaceBeg, kSpaceSize))); - MapWithCallback(kSpaceEnd, AdditionalSize()); - } - - void MapWithCallback(uptr beg, uptr size) { - CHECK_EQ(beg, reinterpret_cast<uptr>(MmapFixedOrDie(beg, size))); - MapUnmapCallback().OnMap(beg, size); - } - - void UnmapWithCallback(uptr beg, uptr size) { - MapUnmapCallback().OnUnmap(beg, size); - UnmapOrDie(reinterpret_cast<void *>(beg), size); - } - - static bool CanAllocate(uptr size, uptr alignment) { - return size <= SizeClassMap::kMaxSize && - alignment <= SizeClassMap::kMaxSize; - } - - NOINLINE Batch* AllocateBatch(AllocatorStats *stat, AllocatorCache *c, - uptr class_id) { - CHECK_LT(class_id, kNumClasses); - RegionInfo *region = GetRegionInfo(class_id); - Batch *b = region->free_list.Pop(); - if (!b) - b = PopulateFreeList(stat, c, class_id, region); - region->n_allocated += b->count; - return b; - } - - NOINLINE void DeallocateBatch(AllocatorStats *stat, uptr class_id, Batch *b) { - RegionInfo *region = GetRegionInfo(class_id); - CHECK_GT(b->count, 0); - region->free_list.Push(b); - region->n_freed += b->count; - } - - static bool PointerIsMine(const void *p) { - return reinterpret_cast<uptr>(p) / kSpaceSize == kSpaceBeg / kSpaceSize; - } - - static uptr GetSizeClass(const void *p) { - return (reinterpret_cast<uptr>(p) / kRegionSize) % kNumClassesRounded; - } - - void *GetBlockBegin(const void *p) { - uptr class_id = GetSizeClass(p); - uptr size = SizeClassMap::Size(class_id); - if (!size) return nullptr; - uptr chunk_idx = GetChunkIdx((uptr)p, size); - uptr reg_beg = (uptr)p & ~(kRegionSize - 1); - uptr beg = chunk_idx * size; - uptr next_beg = beg + size; - if (class_id >= kNumClasses) return nullptr; - RegionInfo *region = GetRegionInfo(class_id); - if (region->mapped_user >= next_beg) - return reinterpret_cast<void*>(reg_beg + beg); - return nullptr; - } - - static uptr GetActuallyAllocatedSize(void *p) { - CHECK(PointerIsMine(p)); - return SizeClassMap::Size(GetSizeClass(p)); - } - - uptr ClassID(uptr size) { return SizeClassMap::ClassID(size); } - - void *GetMetaData(const void *p) { - uptr class_id = GetSizeClass(p); - uptr size = SizeClassMap::Size(class_id); - uptr chunk_idx = GetChunkIdx(reinterpret_cast<uptr>(p), size); - return reinterpret_cast<void*>(kSpaceBeg + (kRegionSize * (class_id + 1)) - - (1 + chunk_idx) * kMetadataSize); - } - - uptr TotalMemoryUsed() { - uptr res = 0; - for (uptr i = 0; i < kNumClasses; i++) - res += GetRegionInfo(i)->allocated_user; - return res; - } - - // Test-only. - void TestOnlyUnmap() { - UnmapWithCallback(kSpaceBeg, kSpaceSize + AdditionalSize()); - } - - void PrintStats() { - uptr total_mapped = 0; - uptr n_allocated = 0; - uptr n_freed = 0; - for (uptr class_id = 1; class_id < kNumClasses; class_id++) { - RegionInfo *region = GetRegionInfo(class_id); - total_mapped += region->mapped_user; - n_allocated += region->n_allocated; - n_freed += region->n_freed; - } - Printf("Stats: SizeClassAllocator64: %zdM mapped in %zd allocations; " - "remains %zd\n", - total_mapped >> 20, n_allocated, n_allocated - n_freed); - for (uptr class_id = 1; class_id < kNumClasses; class_id++) { - RegionInfo *region = GetRegionInfo(class_id); - if (region->mapped_user == 0) continue; - Printf(" %02zd (%zd): total: %zd K allocs: %zd remains: %zd\n", - class_id, - SizeClassMap::Size(class_id), - region->mapped_user >> 10, - region->n_allocated, - region->n_allocated - region->n_freed); - } - } - - // ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone - // introspection API. - void ForceLock() { - for (uptr i = 0; i < kNumClasses; i++) { - GetRegionInfo(i)->mutex.Lock(); - } - } - - void ForceUnlock() { - for (int i = (int)kNumClasses - 1; i >= 0; i--) { - GetRegionInfo(i)->mutex.Unlock(); - } - } - - // Iterate over all existing chunks. - // The allocator must be locked when calling this function. - void ForEachChunk(ForEachChunkCallback callback, void *arg) { - for (uptr class_id = 1; class_id < kNumClasses; class_id++) { - RegionInfo *region = GetRegionInfo(class_id); - uptr chunk_size = SizeClassMap::Size(class_id); - uptr region_beg = kSpaceBeg + class_id * kRegionSize; - for (uptr chunk = region_beg; - chunk < region_beg + region->allocated_user; - chunk += chunk_size) { - // Too slow: CHECK_EQ((void *)chunk, GetBlockBegin((void *)chunk)); - callback(chunk, arg); - } - } - } - - static uptr AdditionalSize() { - return RoundUpTo(sizeof(RegionInfo) * kNumClassesRounded, - GetPageSizeCached()); - } - - typedef SizeClassMap SizeClassMapT; - static const uptr kNumClasses = SizeClassMap::kNumClasses; - static const uptr kNumClassesRounded = SizeClassMap::kNumClassesRounded; - - private: - static const uptr kRegionSize = kSpaceSize / kNumClassesRounded; - static const uptr kSpaceEnd = kSpaceBeg + kSpaceSize; - COMPILER_CHECK(kSpaceBeg % kSpaceSize == 0); - // kRegionSize must be >= 2^32. - COMPILER_CHECK((kRegionSize) >= (1ULL << (SANITIZER_WORDSIZE / 2))); - // Populate the free list with at most this number of bytes at once - // or with one element if its size is greater. - static const uptr kPopulateSize = 1 << 14; - // Call mmap for user memory with at least this size. - static const uptr kUserMapSize = 1 << 16; - // Call mmap for metadata memory with at least this size. - static const uptr kMetaMapSize = 1 << 16; - - struct RegionInfo { - BlockingMutex mutex; - LFStack<Batch> free_list; - uptr allocated_user; // Bytes allocated for user memory. - uptr allocated_meta; // Bytes allocated for metadata. - uptr mapped_user; // Bytes mapped for user memory. - uptr mapped_meta; // Bytes mapped for metadata. - uptr n_allocated, n_freed; // Just stats. - }; - COMPILER_CHECK(sizeof(RegionInfo) >= kCacheLineSize); - - RegionInfo *GetRegionInfo(uptr class_id) { - CHECK_LT(class_id, kNumClasses); - RegionInfo *regions = reinterpret_cast<RegionInfo*>(kSpaceBeg + kSpaceSize); - return ®ions[class_id]; - } - - static uptr GetChunkIdx(uptr chunk, uptr size) { - uptr offset = chunk % kRegionSize; - // Here we divide by a non-constant. This is costly. - // size always fits into 32-bits. If the offset fits too, use 32-bit div. - if (offset >> (SANITIZER_WORDSIZE / 2)) - return offset / size; - return (u32)offset / (u32)size; - } - - NOINLINE Batch* PopulateFreeList(AllocatorStats *stat, AllocatorCache *c, - uptr class_id, RegionInfo *region) { - BlockingMutexLock l(®ion->mutex); - Batch *b = region->free_list.Pop(); - if (b) - return b; - uptr size = SizeClassMap::Size(class_id); - uptr count = size < kPopulateSize ? SizeClassMap::MaxCached(class_id) : 1; - uptr beg_idx = region->allocated_user; - uptr end_idx = beg_idx + count * size; - uptr region_beg = kSpaceBeg + kRegionSize * class_id; - if (end_idx + size > region->mapped_user) { - // Do the mmap for the user memory. - uptr map_size = kUserMapSize; - while (end_idx + size > region->mapped_user + map_size) - map_size += kUserMapSize; - CHECK_GE(region->mapped_user + map_size, end_idx); - MapWithCallback(region_beg + region->mapped_user, map_size); - stat->Add(AllocatorStatMapped, map_size); - region->mapped_user += map_size; - } - uptr total_count = (region->mapped_user - beg_idx - size) - / size / count * count; - region->allocated_meta += total_count * kMetadataSize; - if (region->allocated_meta > region->mapped_meta) { - uptr map_size = kMetaMapSize; - while (region->allocated_meta > region->mapped_meta + map_size) - map_size += kMetaMapSize; - // Do the mmap for the metadata. - CHECK_GE(region->mapped_meta + map_size, region->allocated_meta); - MapWithCallback(region_beg + kRegionSize - - region->mapped_meta - map_size, map_size); - region->mapped_meta += map_size; - } - CHECK_LE(region->allocated_meta, region->mapped_meta); - if (region->mapped_user + region->mapped_meta > kRegionSize) { - Printf("%s: Out of memory. Dying. ", SanitizerToolName); - Printf("The process has exhausted %zuMB for size class %zu.\n", - kRegionSize / 1024 / 1024, size); - Die(); - } - for (;;) { - if (SizeClassMap::SizeClassRequiresSeparateTransferBatch(class_id)) - b = (Batch*)c->Allocate(this, SizeClassMap::ClassID(sizeof(Batch))); - else - b = (Batch*)(region_beg + beg_idx); - b->count = count; - for (uptr i = 0; i < count; i++) - b->batch[i] = (void*)(region_beg + beg_idx + i * size); - region->allocated_user += count * size; - CHECK_LE(region->allocated_user, region->mapped_user); - beg_idx += count * size; - if (beg_idx + count * size + size > region->mapped_user) - break; - CHECK_GT(b->count, 0); - region->free_list.Push(b); - } - return b; - } -}; - -// Maps integers in rage [0, kSize) to u8 values. -template<u64 kSize> -class FlatByteMap { - public: - void TestOnlyInit() { - internal_memset(map_, 0, sizeof(map_)); - } - - void set(uptr idx, u8 val) { - CHECK_LT(idx, kSize); - CHECK_EQ(0U, map_[idx]); - map_[idx] = val; - } - u8 operator[] (uptr idx) { - CHECK_LT(idx, kSize); - // FIXME: CHECK may be too expensive here. - return map_[idx]; - } - private: - u8 map_[kSize]; -}; - -// TwoLevelByteMap maps integers in range [0, kSize1*kSize2) to u8 values. -// It is implemented as a two-dimensional array: array of kSize1 pointers -// to kSize2-byte arrays. The secondary arrays are mmaped on demand. -// Each value is initially zero and can be set to something else only once. -// Setting and getting values from multiple threads is safe w/o extra locking. -template <u64 kSize1, u64 kSize2, class MapUnmapCallback = NoOpMapUnmapCallback> -class TwoLevelByteMap { - public: - void TestOnlyInit() { - internal_memset(map1_, 0, sizeof(map1_)); - mu_.Init(); - } - - void TestOnlyUnmap() { - for (uptr i = 0; i < kSize1; i++) { - u8 *p = Get(i); - if (!p) continue; - MapUnmapCallback().OnUnmap(reinterpret_cast<uptr>(p), kSize2); - UnmapOrDie(p, kSize2); - } - } - - uptr size() const { return kSize1 * kSize2; } - uptr size1() const { return kSize1; } - uptr size2() const { return kSize2; } - - void set(uptr idx, u8 val) { - CHECK_LT(idx, kSize1 * kSize2); - u8 *map2 = GetOrCreate(idx / kSize2); - CHECK_EQ(0U, map2[idx % kSize2]); - map2[idx % kSize2] = val; - } - - u8 operator[] (uptr idx) const { - CHECK_LT(idx, kSize1 * kSize2); - u8 *map2 = Get(idx / kSize2); - if (!map2) return 0; - return map2[idx % kSize2]; - } - - private: - u8 *Get(uptr idx) const { - CHECK_LT(idx, kSize1); - return reinterpret_cast<u8 *>( - atomic_load(&map1_[idx], memory_order_acquire)); - } - - u8 *GetOrCreate(uptr idx) { - u8 *res = Get(idx); - if (!res) { - SpinMutexLock l(&mu_); - if (!(res = Get(idx))) { - res = (u8*)MmapOrDie(kSize2, "TwoLevelByteMap"); - MapUnmapCallback().OnMap(reinterpret_cast<uptr>(res), kSize2); - atomic_store(&map1_[idx], reinterpret_cast<uptr>(res), - memory_order_release); - } - } - return res; - } - - atomic_uintptr_t map1_[kSize1]; - StaticSpinMutex mu_; -}; - -// SizeClassAllocator32 -- allocator for 32-bit address space. -// This allocator can theoretically be used on 64-bit arch, but there it is less -// efficient than SizeClassAllocator64. -// -// [kSpaceBeg, kSpaceBeg + kSpaceSize) is the range of addresses which can -// be returned by MmapOrDie(). -// -// Region: -// a result of a single call to MmapAlignedOrDie(kRegionSize, kRegionSize). -// Since the regions are aligned by kRegionSize, there are exactly -// kNumPossibleRegions possible regions in the address space and so we keep -// a ByteMap possible_regions to store the size classes of each Region. -// 0 size class means the region is not used by the allocator. -// -// One Region is used to allocate chunks of a single size class. -// A Region looks like this: -// UserChunk1 .. UserChunkN <gap> MetaChunkN .. MetaChunk1 -// -// In order to avoid false sharing the objects of this class should be -// chache-line aligned. -template <const uptr kSpaceBeg, const u64 kSpaceSize, - const uptr kMetadataSize, class SizeClassMap, - const uptr kRegionSizeLog, - class ByteMap, - class MapUnmapCallback = NoOpMapUnmapCallback> -class SizeClassAllocator32 { - public: - typedef typename SizeClassMap::TransferBatch Batch; - typedef SizeClassAllocator32<kSpaceBeg, kSpaceSize, kMetadataSize, - SizeClassMap, kRegionSizeLog, ByteMap, MapUnmapCallback> ThisT; - typedef SizeClassAllocatorLocalCache<ThisT> AllocatorCache; - - void Init() { - possible_regions.TestOnlyInit(); - internal_memset(size_class_info_array, 0, sizeof(size_class_info_array)); - } - - void *MapWithCallback(uptr size) { - size = RoundUpTo(size, GetPageSizeCached()); - void *res = MmapOrDie(size, "SizeClassAllocator32"); - MapUnmapCallback().OnMap((uptr)res, size); - return res; - } - - void UnmapWithCallback(uptr beg, uptr size) { - MapUnmapCallback().OnUnmap(beg, size); - UnmapOrDie(reinterpret_cast<void *>(beg), size); - } - - static bool CanAllocate(uptr size, uptr alignment) { - return size <= SizeClassMap::kMaxSize && - alignment <= SizeClassMap::kMaxSize; - } - - void *GetMetaData(const void *p) { - CHECK(PointerIsMine(p)); - uptr mem = reinterpret_cast<uptr>(p); - uptr beg = ComputeRegionBeg(mem); - uptr size = SizeClassMap::Size(GetSizeClass(p)); - u32 offset = mem - beg; - uptr n = offset / (u32)size; // 32-bit division - uptr meta = (beg + kRegionSize) - (n + 1) * kMetadataSize; - return reinterpret_cast<void*>(meta); - } - - NOINLINE Batch* AllocateBatch(AllocatorStats *stat, AllocatorCache *c, - uptr class_id) { - CHECK_LT(class_id, kNumClasses); - SizeClassInfo *sci = GetSizeClassInfo(class_id); - SpinMutexLock l(&sci->mutex); - if (sci->free_list.empty()) - PopulateFreeList(stat, c, sci, class_id); - CHECK(!sci->free_list.empty()); - Batch *b = sci->free_list.front(); - sci->free_list.pop_front(); - return b; - } - - NOINLINE void DeallocateBatch(AllocatorStats *stat, uptr class_id, Batch *b) { - CHECK_LT(class_id, kNumClasses); - SizeClassInfo *sci = GetSizeClassInfo(class_id); - SpinMutexLock l(&sci->mutex); - CHECK_GT(b->count, 0); - sci->free_list.push_front(b); - } - - bool PointerIsMine(const void *p) { - return GetSizeClass(p) != 0; - } - - uptr GetSizeClass(const void *p) { - return possible_regions[ComputeRegionId(reinterpret_cast<uptr>(p))]; - } - - void *GetBlockBegin(const void *p) { - CHECK(PointerIsMine(p)); - uptr mem = reinterpret_cast<uptr>(p); - uptr beg = ComputeRegionBeg(mem); - uptr size = SizeClassMap::Size(GetSizeClass(p)); - u32 offset = mem - beg; - u32 n = offset / (u32)size; // 32-bit division - uptr res = beg + (n * (u32)size); - return reinterpret_cast<void*>(res); - } - - uptr GetActuallyAllocatedSize(void *p) { - CHECK(PointerIsMine(p)); - return SizeClassMap::Size(GetSizeClass(p)); - } - - uptr ClassID(uptr size) { return SizeClassMap::ClassID(size); } - - uptr TotalMemoryUsed() { - // No need to lock here. - uptr res = 0; - for (uptr i = 0; i < kNumPossibleRegions; i++) - if (possible_regions[i]) - res += kRegionSize; - return res; - } - - void TestOnlyUnmap() { - for (uptr i = 0; i < kNumPossibleRegions; i++) - if (possible_regions[i]) - UnmapWithCallback((i * kRegionSize), kRegionSize); - } - - // ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone - // introspection API. - void ForceLock() { - for (uptr i = 0; i < kNumClasses; i++) { - GetSizeClassInfo(i)->mutex.Lock(); - } - } - - void ForceUnlock() { - for (int i = kNumClasses - 1; i >= 0; i--) { - GetSizeClassInfo(i)->mutex.Unlock(); - } - } - - // Iterate over all existing chunks. - // The allocator must be locked when calling this function. - void ForEachChunk(ForEachChunkCallback callback, void *arg) { - for (uptr region = 0; region < kNumPossibleRegions; region++) - if (possible_regions[region]) { - uptr chunk_size = SizeClassMap::Size(possible_regions[region]); - uptr max_chunks_in_region = kRegionSize / (chunk_size + kMetadataSize); - uptr region_beg = region * kRegionSize; - for (uptr chunk = region_beg; - chunk < region_beg + max_chunks_in_region * chunk_size; - chunk += chunk_size) { - // Too slow: CHECK_EQ((void *)chunk, GetBlockBegin((void *)chunk)); - callback(chunk, arg); - } - } - } - - void PrintStats() { - } - - static uptr AdditionalSize() { - return 0; - } - - typedef SizeClassMap SizeClassMapT; - static const uptr kNumClasses = SizeClassMap::kNumClasses; - - private: - static const uptr kRegionSize = 1 << kRegionSizeLog; - static const uptr kNumPossibleRegions = kSpaceSize / kRegionSize; - - struct SizeClassInfo { - SpinMutex mutex; - IntrusiveList<Batch> free_list; - char padding[kCacheLineSize - sizeof(uptr) - sizeof(IntrusiveList<Batch>)]; - }; - COMPILER_CHECK(sizeof(SizeClassInfo) == kCacheLineSize); - - uptr ComputeRegionId(uptr mem) { - uptr res = mem >> kRegionSizeLog; - CHECK_LT(res, kNumPossibleRegions); - return res; - } - - uptr ComputeRegionBeg(uptr mem) { - return mem & ~(kRegionSize - 1); - } - - uptr AllocateRegion(AllocatorStats *stat, uptr class_id) { - CHECK_LT(class_id, kNumClasses); - uptr res = reinterpret_cast<uptr>(MmapAlignedOrDie(kRegionSize, kRegionSize, - "SizeClassAllocator32")); - MapUnmapCallback().OnMap(res, kRegionSize); - stat->Add(AllocatorStatMapped, kRegionSize); - CHECK_EQ(0U, (res & (kRegionSize - 1))); - possible_regions.set(ComputeRegionId(res), static_cast<u8>(class_id)); - return res; - } - - SizeClassInfo *GetSizeClassInfo(uptr class_id) { - CHECK_LT(class_id, kNumClasses); - return &size_class_info_array[class_id]; - } - - void PopulateFreeList(AllocatorStats *stat, AllocatorCache *c, - SizeClassInfo *sci, uptr class_id) { - uptr size = SizeClassMap::Size(class_id); - uptr reg = AllocateRegion(stat, class_id); - uptr n_chunks = kRegionSize / (size + kMetadataSize); - uptr max_count = SizeClassMap::MaxCached(class_id); - Batch *b = nullptr; - for (uptr i = reg; i < reg + n_chunks * size; i += size) { - if (!b) { - if (SizeClassMap::SizeClassRequiresSeparateTransferBatch(class_id)) - b = (Batch*)c->Allocate(this, SizeClassMap::ClassID(sizeof(Batch))); - else - b = (Batch*)i; - b->count = 0; - } - b->batch[b->count++] = (void*)i; - if (b->count == max_count) { - CHECK_GT(b->count, 0); - sci->free_list.push_back(b); - b = nullptr; - } - } - if (b) { - CHECK_GT(b->count, 0); - sci->free_list.push_back(b); - } - } - - ByteMap possible_regions; - SizeClassInfo size_class_info_array[kNumClasses]; -}; - -// Objects of this type should be used as local caches for SizeClassAllocator64 -// or SizeClassAllocator32. Since the typical use of this class is to have one -// object per thread in TLS, is has to be POD. -template<class SizeClassAllocator> -struct SizeClassAllocatorLocalCache { - typedef SizeClassAllocator Allocator; - static const uptr kNumClasses = SizeClassAllocator::kNumClasses; - - void Init(AllocatorGlobalStats *s) { - stats_.Init(); - if (s) - s->Register(&stats_); - } - - void Destroy(SizeClassAllocator *allocator, AllocatorGlobalStats *s) { - Drain(allocator); - if (s) - s->Unregister(&stats_); - } - - void *Allocate(SizeClassAllocator *allocator, uptr class_id) { - CHECK_NE(class_id, 0UL); - CHECK_LT(class_id, kNumClasses); - stats_.Add(AllocatorStatAllocated, SizeClassMap::Size(class_id)); - PerClass *c = &per_class_[class_id]; - if (UNLIKELY(c->count == 0)) - Refill(allocator, class_id); - void *res = c->batch[--c->count]; - PREFETCH(c->batch[c->count - 1]); - return res; - } - - void Deallocate(SizeClassAllocator *allocator, uptr class_id, void *p) { - CHECK_NE(class_id, 0UL); - CHECK_LT(class_id, kNumClasses); - // If the first allocator call on a new thread is a deallocation, then - // max_count will be zero, leading to check failure. - InitCache(); - stats_.Sub(AllocatorStatAllocated, SizeClassMap::Size(class_id)); - PerClass *c = &per_class_[class_id]; - CHECK_NE(c->max_count, 0UL); - if (UNLIKELY(c->count == c->max_count)) - Drain(allocator, class_id); - c->batch[c->count++] = p; - } - - void Drain(SizeClassAllocator *allocator) { - for (uptr class_id = 0; class_id < kNumClasses; class_id++) { - PerClass *c = &per_class_[class_id]; - while (c->count > 0) - Drain(allocator, class_id); - } - } - - // private: - typedef typename SizeClassAllocator::SizeClassMapT SizeClassMap; - typedef typename SizeClassMap::TransferBatch Batch; - struct PerClass { - uptr count; - uptr max_count; - void *batch[2 * SizeClassMap::kMaxNumCached]; - }; - PerClass per_class_[kNumClasses]; - AllocatorStats stats_; - - void InitCache() { - if (per_class_[1].max_count) - return; - for (uptr i = 0; i < kNumClasses; i++) { - PerClass *c = &per_class_[i]; - c->max_count = 2 * SizeClassMap::MaxCached(i); - } - } - - NOINLINE void Refill(SizeClassAllocator *allocator, uptr class_id) { - InitCache(); - PerClass *c = &per_class_[class_id]; - Batch *b = allocator->AllocateBatch(&stats_, this, class_id); - CHECK_GT(b->count, 0); - for (uptr i = 0; i < b->count; i++) - c->batch[i] = b->batch[i]; - c->count = b->count; - if (SizeClassMap::SizeClassRequiresSeparateTransferBatch(class_id)) - Deallocate(allocator, SizeClassMap::ClassID(sizeof(Batch)), b); - } - - NOINLINE void Drain(SizeClassAllocator *allocator, uptr class_id) { - InitCache(); - PerClass *c = &per_class_[class_id]; - Batch *b; - if (SizeClassMap::SizeClassRequiresSeparateTransferBatch(class_id)) - b = (Batch*)Allocate(allocator, SizeClassMap::ClassID(sizeof(Batch))); - else - b = (Batch*)c->batch[0]; - uptr cnt = Min(c->max_count / 2, c->count); - for (uptr i = 0; i < cnt; i++) { - b->batch[i] = c->batch[i]; - c->batch[i] = c->batch[i + c->max_count / 2]; - } - b->count = cnt; - c->count -= cnt; - CHECK_GT(b->count, 0); - allocator->DeallocateBatch(&stats_, class_id, b); - } -}; - -// This class can (de)allocate only large chunks of memory using mmap/unmap. -// The main purpose of this allocator is to cover large and rare allocation -// sizes not covered by more efficient allocators (e.g. SizeClassAllocator64). -template <class MapUnmapCallback = NoOpMapUnmapCallback> -class LargeMmapAllocator { - public: - void InitLinkerInitialized(bool may_return_null) { - page_size_ = GetPageSizeCached(); - atomic_store(&may_return_null_, may_return_null, memory_order_relaxed); - } - - void Init(bool may_return_null) { - internal_memset(this, 0, sizeof(*this)); - InitLinkerInitialized(may_return_null); - } - - void *Allocate(AllocatorStats *stat, uptr size, uptr alignment) { - CHECK(IsPowerOfTwo(alignment)); - uptr map_size = RoundUpMapSize(size); - if (alignment > page_size_) - map_size += alignment; - // Overflow. - if (map_size < size) - return ReturnNullOrDie(); - uptr map_beg = reinterpret_cast<uptr>( - MmapOrDie(map_size, "LargeMmapAllocator")); - CHECK(IsAligned(map_beg, page_size_)); - MapUnmapCallback().OnMap(map_beg, map_size); - uptr map_end = map_beg + map_size; - uptr res = map_beg + page_size_; - if (res & (alignment - 1)) // Align. - res += alignment - (res & (alignment - 1)); - CHECK(IsAligned(res, alignment)); - CHECK(IsAligned(res, page_size_)); - CHECK_GE(res + size, map_beg); - CHECK_LE(res + size, map_end); - Header *h = GetHeader(res); - h->size = size; - h->map_beg = map_beg; - h->map_size = map_size; - uptr size_log = MostSignificantSetBitIndex(map_size); - CHECK_LT(size_log, ARRAY_SIZE(stats.by_size_log)); - { - SpinMutexLock l(&mutex_); - uptr idx = n_chunks_++; - chunks_sorted_ = false; - CHECK_LT(idx, kMaxNumChunks); - h->chunk_idx = idx; - chunks_[idx] = h; - stats.n_allocs++; - stats.currently_allocated += map_size; - stats.max_allocated = Max(stats.max_allocated, stats.currently_allocated); - stats.by_size_log[size_log]++; - stat->Add(AllocatorStatAllocated, map_size); - stat->Add(AllocatorStatMapped, map_size); - } - return reinterpret_cast<void*>(res); - } - - void *ReturnNullOrDie() { - if (atomic_load(&may_return_null_, memory_order_acquire)) - return nullptr; - ReportAllocatorCannotReturnNull(); - } - - void SetMayReturnNull(bool may_return_null) { - atomic_store(&may_return_null_, may_return_null, memory_order_release); - } - - void Deallocate(AllocatorStats *stat, void *p) { - Header *h = GetHeader(p); - { - SpinMutexLock l(&mutex_); - uptr idx = h->chunk_idx; - CHECK_EQ(chunks_[idx], h); - CHECK_LT(idx, n_chunks_); - chunks_[idx] = chunks_[n_chunks_ - 1]; - chunks_[idx]->chunk_idx = idx; - n_chunks_--; - chunks_sorted_ = false; - stats.n_frees++; - stats.currently_allocated -= h->map_size; - stat->Sub(AllocatorStatAllocated, h->map_size); - stat->Sub(AllocatorStatMapped, h->map_size); - } - MapUnmapCallback().OnUnmap(h->map_beg, h->map_size); - UnmapOrDie(reinterpret_cast<void*>(h->map_beg), h->map_size); - } - - uptr TotalMemoryUsed() { - SpinMutexLock l(&mutex_); - uptr res = 0; - for (uptr i = 0; i < n_chunks_; i++) { - Header *h = chunks_[i]; - CHECK_EQ(h->chunk_idx, i); - res += RoundUpMapSize(h->size); - } - return res; - } - - bool PointerIsMine(const void *p) { - return GetBlockBegin(p) != nullptr; - } - - uptr GetActuallyAllocatedSize(void *p) { - return RoundUpTo(GetHeader(p)->size, page_size_); - } - - // At least page_size_/2 metadata bytes is available. - void *GetMetaData(const void *p) { - // Too slow: CHECK_EQ(p, GetBlockBegin(p)); - if (!IsAligned(reinterpret_cast<uptr>(p), page_size_)) { - Printf("%s: bad pointer %p\n", SanitizerToolName, p); - CHECK(IsAligned(reinterpret_cast<uptr>(p), page_size_)); - } - return GetHeader(p) + 1; - } - - void *GetBlockBegin(const void *ptr) { - uptr p = reinterpret_cast<uptr>(ptr); - SpinMutexLock l(&mutex_); - uptr nearest_chunk = 0; - // Cache-friendly linear search. - for (uptr i = 0; i < n_chunks_; i++) { - uptr ch = reinterpret_cast<uptr>(chunks_[i]); - if (p < ch) continue; // p is at left to this chunk, skip it. - if (p - ch < p - nearest_chunk) - nearest_chunk = ch; - } - if (!nearest_chunk) - return nullptr; - Header *h = reinterpret_cast<Header *>(nearest_chunk); - CHECK_GE(nearest_chunk, h->map_beg); - CHECK_LT(nearest_chunk, h->map_beg + h->map_size); - CHECK_LE(nearest_chunk, p); - if (h->map_beg + h->map_size <= p) - return nullptr; - return GetUser(h); - } - - // This function does the same as GetBlockBegin, but is much faster. - // Must be called with the allocator locked. - void *GetBlockBeginFastLocked(void *ptr) { - mutex_.CheckLocked(); - uptr p = reinterpret_cast<uptr>(ptr); - uptr n = n_chunks_; - if (!n) return nullptr; - if (!chunks_sorted_) { - // Do one-time sort. chunks_sorted_ is reset in Allocate/Deallocate. - SortArray(reinterpret_cast<uptr*>(chunks_), n); - for (uptr i = 0; i < n; i++) - chunks_[i]->chunk_idx = i; - chunks_sorted_ = true; - min_mmap_ = reinterpret_cast<uptr>(chunks_[0]); - max_mmap_ = reinterpret_cast<uptr>(chunks_[n - 1]) + - chunks_[n - 1]->map_size; - } - if (p < min_mmap_ || p >= max_mmap_) - return nullptr; - uptr beg = 0, end = n - 1; - // This loop is a log(n) lower_bound. It does not check for the exact match - // to avoid expensive cache-thrashing loads. - while (end - beg >= 2) { - uptr mid = (beg + end) / 2; // Invariant: mid >= beg + 1 - if (p < reinterpret_cast<uptr>(chunks_[mid])) - end = mid - 1; // We are not interested in chunks_[mid]. - else - beg = mid; // chunks_[mid] may still be what we want. - } - - if (beg < end) { - CHECK_EQ(beg + 1, end); - // There are 2 chunks left, choose one. - if (p >= reinterpret_cast<uptr>(chunks_[end])) - beg = end; - } - - Header *h = chunks_[beg]; - if (h->map_beg + h->map_size <= p || p < h->map_beg) - return nullptr; - return GetUser(h); - } - - void PrintStats() { - Printf("Stats: LargeMmapAllocator: allocated %zd times, " - "remains %zd (%zd K) max %zd M; by size logs: ", - stats.n_allocs, stats.n_allocs - stats.n_frees, - stats.currently_allocated >> 10, stats.max_allocated >> 20); - for (uptr i = 0; i < ARRAY_SIZE(stats.by_size_log); i++) { - uptr c = stats.by_size_log[i]; - if (!c) continue; - Printf("%zd:%zd; ", i, c); - } - Printf("\n"); - } - - // ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone - // introspection API. - void ForceLock() { - mutex_.Lock(); - } - - void ForceUnlock() { - mutex_.Unlock(); - } - - // Iterate over all existing chunks. - // The allocator must be locked when calling this function. - void ForEachChunk(ForEachChunkCallback callback, void *arg) { - for (uptr i = 0; i < n_chunks_; i++) - callback(reinterpret_cast<uptr>(GetUser(chunks_[i])), arg); - } - - private: - static const int kMaxNumChunks = 1 << FIRST_32_SECOND_64(15, 18); - struct Header { - uptr map_beg; - uptr map_size; - uptr size; - uptr chunk_idx; - }; - - Header *GetHeader(uptr p) { - CHECK(IsAligned(p, page_size_)); - return reinterpret_cast<Header*>(p - page_size_); - } - Header *GetHeader(const void *p) { - return GetHeader(reinterpret_cast<uptr>(p)); - } - - void *GetUser(Header *h) { - CHECK(IsAligned((uptr)h, page_size_)); - return reinterpret_cast<void*>(reinterpret_cast<uptr>(h) + page_size_); - } - - uptr RoundUpMapSize(uptr size) { - return RoundUpTo(size, page_size_) + page_size_; - } - - uptr page_size_; - Header *chunks_[kMaxNumChunks]; - uptr n_chunks_; - uptr min_mmap_, max_mmap_; - bool chunks_sorted_; - struct Stats { - uptr n_allocs, n_frees, currently_allocated, max_allocated, by_size_log[64]; - } stats; - atomic_uint8_t may_return_null_; - SpinMutex mutex_; -}; - -// This class implements a complete memory allocator by using two -// internal allocators: -// PrimaryAllocator is efficient, but may not allocate some sizes (alignments). -// When allocating 2^x bytes it should return 2^x aligned chunk. -// PrimaryAllocator is used via a local AllocatorCache. -// SecondaryAllocator can allocate anything, but is not efficient. -template <class PrimaryAllocator, class AllocatorCache, - class SecondaryAllocator> // NOLINT -class CombinedAllocator { - public: - void InitCommon(bool may_return_null) { - primary_.Init(); - atomic_store(&may_return_null_, may_return_null, memory_order_relaxed); - } - - void InitLinkerInitialized(bool may_return_null) { - secondary_.InitLinkerInitialized(may_return_null); - stats_.InitLinkerInitialized(); - InitCommon(may_return_null); - } - - void Init(bool may_return_null) { - secondary_.Init(may_return_null); - stats_.Init(); - InitCommon(may_return_null); - } - - void *Allocate(AllocatorCache *cache, uptr size, uptr alignment, - bool cleared = false, bool check_rss_limit = false) { - // Returning 0 on malloc(0) may break a lot of code. - if (size == 0) - size = 1; - if (size + alignment < size) - return ReturnNullOrDie(); - if (check_rss_limit && RssLimitIsExceeded()) - return ReturnNullOrDie(); - if (alignment > 8) - size = RoundUpTo(size, alignment); - void *res; - bool from_primary = primary_.CanAllocate(size, alignment); - if (from_primary) - res = cache->Allocate(&primary_, primary_.ClassID(size)); - else - res = secondary_.Allocate(&stats_, size, alignment); - if (alignment > 8) - CHECK_EQ(reinterpret_cast<uptr>(res) & (alignment - 1), 0); - if (cleared && res && from_primary) - internal_bzero_aligned16(res, RoundUpTo(size, 16)); - return res; - } - - bool MayReturnNull() const { - return atomic_load(&may_return_null_, memory_order_acquire); - } - - void *ReturnNullOrDie() { - if (MayReturnNull()) - return nullptr; - ReportAllocatorCannotReturnNull(); - } - - void SetMayReturnNull(bool may_return_null) { - secondary_.SetMayReturnNull(may_return_null); - atomic_store(&may_return_null_, may_return_null, memory_order_release); - } - - bool RssLimitIsExceeded() { - return atomic_load(&rss_limit_is_exceeded_, memory_order_acquire); - } - - void SetRssLimitIsExceeded(bool rss_limit_is_exceeded) { - atomic_store(&rss_limit_is_exceeded_, rss_limit_is_exceeded, - memory_order_release); - } - - void Deallocate(AllocatorCache *cache, void *p) { - if (!p) return; - if (primary_.PointerIsMine(p)) - cache->Deallocate(&primary_, primary_.GetSizeClass(p), p); - else - secondary_.Deallocate(&stats_, p); - } - - void *Reallocate(AllocatorCache *cache, void *p, uptr new_size, - uptr alignment) { - if (!p) - return Allocate(cache, new_size, alignment); - if (!new_size) { - Deallocate(cache, p); - return nullptr; - } - CHECK(PointerIsMine(p)); - uptr old_size = GetActuallyAllocatedSize(p); - uptr memcpy_size = Min(new_size, old_size); - void *new_p = Allocate(cache, new_size, alignment); - if (new_p) - internal_memcpy(new_p, p, memcpy_size); - Deallocate(cache, p); - return new_p; - } - - bool PointerIsMine(void *p) { - if (primary_.PointerIsMine(p)) - return true; - return secondary_.PointerIsMine(p); - } - - bool FromPrimary(void *p) { - return primary_.PointerIsMine(p); - } - - void *GetMetaData(const void *p) { - if (primary_.PointerIsMine(p)) - return primary_.GetMetaData(p); - return secondary_.GetMetaData(p); - } - - void *GetBlockBegin(const void *p) { - if (primary_.PointerIsMine(p)) - return primary_.GetBlockBegin(p); - return secondary_.GetBlockBegin(p); - } - - // This function does the same as GetBlockBegin, but is much faster. - // Must be called with the allocator locked. - void *GetBlockBeginFastLocked(void *p) { - if (primary_.PointerIsMine(p)) - return primary_.GetBlockBegin(p); - return secondary_.GetBlockBeginFastLocked(p); - } - - uptr GetActuallyAllocatedSize(void *p) { - if (primary_.PointerIsMine(p)) - return primary_.GetActuallyAllocatedSize(p); - return secondary_.GetActuallyAllocatedSize(p); - } - - uptr TotalMemoryUsed() { - return primary_.TotalMemoryUsed() + secondary_.TotalMemoryUsed(); - } - - void TestOnlyUnmap() { primary_.TestOnlyUnmap(); } - - void InitCache(AllocatorCache *cache) { - cache->Init(&stats_); - } - - void DestroyCache(AllocatorCache *cache) { - cache->Destroy(&primary_, &stats_); - } - - void SwallowCache(AllocatorCache *cache) { - cache->Drain(&primary_); - } - - void GetStats(AllocatorStatCounters s) const { - stats_.Get(s); - } - - void PrintStats() { - primary_.PrintStats(); - secondary_.PrintStats(); - } - - // ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone - // introspection API. - void ForceLock() { - primary_.ForceLock(); - secondary_.ForceLock(); - } - - void ForceUnlock() { - secondary_.ForceUnlock(); - primary_.ForceUnlock(); - } - - // Iterate over all existing chunks. - // The allocator must be locked when calling this function. - void ForEachChunk(ForEachChunkCallback callback, void *arg) { - primary_.ForEachChunk(callback, arg); - secondary_.ForEachChunk(callback, arg); - } - - private: - PrimaryAllocator primary_; - SecondaryAllocator secondary_; - AllocatorGlobalStats stats_; - atomic_uint8_t may_return_null_; - atomic_uint8_t rss_limit_is_exceeded_; -}; - // Returns true if calloc(size, n) should return 0 due to overflow in size*n. bool CallocShouldReturnNullDueToOverflow(uptr size, uptr n); +#include "sanitizer_allocator_size_class_map.h" +#include "sanitizer_allocator_stats.h" +#include "sanitizer_allocator_primary64.h" +#include "sanitizer_allocator_bytemap.h" +#include "sanitizer_allocator_primary32.h" +#include "sanitizer_allocator_local_cache.h" +#include "sanitizer_allocator_secondary.h" +#include "sanitizer_allocator_combined.h" + } // namespace __sanitizer #endif // SANITIZER_ALLOCATOR_H diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_bytemap.h b/libsanitizer/sanitizer_common/sanitizer_allocator_bytemap.h new file mode 100644 index 000000000000..5e768ce9ef97 --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_allocator_bytemap.h @@ -0,0 +1,100 @@ +//===-- sanitizer_allocator_bytemap.h ---------------------------*- C++ -*-===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Part of the Sanitizer Allocator. +// +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_ALLOCATOR_H +#error This file must be included inside sanitizer_allocator.h +#endif + +// Maps integers in rage [0, kSize) to u8 values. +template<u64 kSize> +class FlatByteMap { + public: + void TestOnlyInit() { + internal_memset(map_, 0, sizeof(map_)); + } + + void set(uptr idx, u8 val) { + CHECK_LT(idx, kSize); + CHECK_EQ(0U, map_[idx]); + map_[idx] = val; + } + u8 operator[] (uptr idx) { + CHECK_LT(idx, kSize); + // FIXME: CHECK may be too expensive here. + return map_[idx]; + } + private: + u8 map_[kSize]; +}; + +// TwoLevelByteMap maps integers in range [0, kSize1*kSize2) to u8 values. +// It is implemented as a two-dimensional array: array of kSize1 pointers +// to kSize2-byte arrays. The secondary arrays are mmaped on demand. +// Each value is initially zero and can be set to something else only once. +// Setting and getting values from multiple threads is safe w/o extra locking. +template <u64 kSize1, u64 kSize2, class MapUnmapCallback = NoOpMapUnmapCallback> +class TwoLevelByteMap { + public: + void TestOnlyInit() { + internal_memset(map1_, 0, sizeof(map1_)); + mu_.Init(); + } + + void TestOnlyUnmap() { + for (uptr i = 0; i < kSize1; i++) { + u8 *p = Get(i); + if (!p) continue; + MapUnmapCallback().OnUnmap(reinterpret_cast<uptr>(p), kSize2); + UnmapOrDie(p, kSize2); + } + } + + uptr size() const { return kSize1 * kSize2; } + uptr size1() const { return kSize1; } + uptr size2() const { return kSize2; } + + void set(uptr idx, u8 val) { + CHECK_LT(idx, kSize1 * kSize2); + u8 *map2 = GetOrCreate(idx / kSize2); + CHECK_EQ(0U, map2[idx % kSize2]); + map2[idx % kSize2] = val; + } + + u8 operator[] (uptr idx) const { + CHECK_LT(idx, kSize1 * kSize2); + u8 *map2 = Get(idx / kSize2); + if (!map2) return 0; + return map2[idx % kSize2]; + } + + private: + u8 *Get(uptr idx) const { + CHECK_LT(idx, kSize1); + return reinterpret_cast<u8 *>( + atomic_load(&map1_[idx], memory_order_acquire)); + } + + u8 *GetOrCreate(uptr idx) { + u8 *res = Get(idx); + if (!res) { + SpinMutexLock l(&mu_); + if (!(res = Get(idx))) { + res = (u8*)MmapOrDie(kSize2, "TwoLevelByteMap"); + MapUnmapCallback().OnMap(reinterpret_cast<uptr>(res), kSize2); + atomic_store(&map1_[idx], reinterpret_cast<uptr>(res), + memory_order_release); + } + } + return res; + } + + atomic_uintptr_t map1_[kSize1]; + StaticSpinMutex mu_; +}; diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h b/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h new file mode 100644 index 000000000000..4dc9ca7401f7 --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h @@ -0,0 +1,209 @@ +//===-- sanitizer_allocator_combined.h --------------------------*- C++ -*-===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Part of the Sanitizer Allocator. +// +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_ALLOCATOR_H +#error This file must be included inside sanitizer_allocator.h +#endif + +// This class implements a complete memory allocator by using two +// internal allocators: +// PrimaryAllocator is efficient, but may not allocate some sizes (alignments). +// When allocating 2^x bytes it should return 2^x aligned chunk. +// PrimaryAllocator is used via a local AllocatorCache. +// SecondaryAllocator can allocate anything, but is not efficient. +template <class PrimaryAllocator, class AllocatorCache, + class SecondaryAllocator> // NOLINT +class CombinedAllocator { + public: + void InitCommon(bool may_return_null) { + primary_.Init(); + atomic_store(&may_return_null_, may_return_null, memory_order_relaxed); + } + + void InitLinkerInitialized(bool may_return_null) { + secondary_.InitLinkerInitialized(may_return_null); + stats_.InitLinkerInitialized(); + InitCommon(may_return_null); + } + + void Init(bool may_return_null) { + secondary_.Init(may_return_null); + stats_.Init(); + InitCommon(may_return_null); + } + + void *Allocate(AllocatorCache *cache, uptr size, uptr alignment, + bool cleared = false, bool check_rss_limit = false) { + // Returning 0 on malloc(0) may break a lot of code. + if (size == 0) + size = 1; + if (size + alignment < size) return ReturnNullOrDieOnBadRequest(); + if (check_rss_limit && RssLimitIsExceeded()) return ReturnNullOrDieOnOOM(); + if (alignment > 8) + size = RoundUpTo(size, alignment); + void *res; + bool from_primary = primary_.CanAllocate(size, alignment); + if (from_primary) + res = cache->Allocate(&primary_, primary_.ClassID(size)); + else + res = secondary_.Allocate(&stats_, size, alignment); + if (alignment > 8) + CHECK_EQ(reinterpret_cast<uptr>(res) & (alignment - 1), 0); + if (cleared && res && from_primary) + internal_bzero_aligned16(res, RoundUpTo(size, 16)); + return res; + } + + bool MayReturnNull() const { + return atomic_load(&may_return_null_, memory_order_acquire); + } + + void *ReturnNullOrDieOnBadRequest() { + if (MayReturnNull()) + return nullptr; + ReportAllocatorCannotReturnNull(false); + } + + void *ReturnNullOrDieOnOOM() { + if (MayReturnNull()) return nullptr; + ReportAllocatorCannotReturnNull(true); + } + + void SetMayReturnNull(bool may_return_null) { + secondary_.SetMayReturnNull(may_return_null); + atomic_store(&may_return_null_, may_return_null, memory_order_release); + } + + bool RssLimitIsExceeded() { + return atomic_load(&rss_limit_is_exceeded_, memory_order_acquire); + } + + void SetRssLimitIsExceeded(bool rss_limit_is_exceeded) { + atomic_store(&rss_limit_is_exceeded_, rss_limit_is_exceeded, + memory_order_release); + } + + void Deallocate(AllocatorCache *cache, void *p) { + if (!p) return; + if (primary_.PointerIsMine(p)) + cache->Deallocate(&primary_, primary_.GetSizeClass(p), p); + else + secondary_.Deallocate(&stats_, p); + } + + void *Reallocate(AllocatorCache *cache, void *p, uptr new_size, + uptr alignment) { + if (!p) + return Allocate(cache, new_size, alignment); + if (!new_size) { + Deallocate(cache, p); + return nullptr; + } + CHECK(PointerIsMine(p)); + uptr old_size = GetActuallyAllocatedSize(p); + uptr memcpy_size = Min(new_size, old_size); + void *new_p = Allocate(cache, new_size, alignment); + if (new_p) + internal_memcpy(new_p, p, memcpy_size); + Deallocate(cache, p); + return new_p; + } + + bool PointerIsMine(void *p) { + if (primary_.PointerIsMine(p)) + return true; + return secondary_.PointerIsMine(p); + } + + bool FromPrimary(void *p) { + return primary_.PointerIsMine(p); + } + + void *GetMetaData(const void *p) { + if (primary_.PointerIsMine(p)) + return primary_.GetMetaData(p); + return secondary_.GetMetaData(p); + } + + void *GetBlockBegin(const void *p) { + if (primary_.PointerIsMine(p)) + return primary_.GetBlockBegin(p); + return secondary_.GetBlockBegin(p); + } + + // This function does the same as GetBlockBegin, but is much faster. + // Must be called with the allocator locked. + void *GetBlockBeginFastLocked(void *p) { + if (primary_.PointerIsMine(p)) + return primary_.GetBlockBegin(p); + return secondary_.GetBlockBeginFastLocked(p); + } + + uptr GetActuallyAllocatedSize(void *p) { + if (primary_.PointerIsMine(p)) + return primary_.GetActuallyAllocatedSize(p); + return secondary_.GetActuallyAllocatedSize(p); + } + + uptr TotalMemoryUsed() { + return primary_.TotalMemoryUsed() + secondary_.TotalMemoryUsed(); + } + + void TestOnlyUnmap() { primary_.TestOnlyUnmap(); } + + void InitCache(AllocatorCache *cache) { + cache->Init(&stats_); + } + + void DestroyCache(AllocatorCache *cache) { + cache->Destroy(&primary_, &stats_); + } + + void SwallowCache(AllocatorCache *cache) { + cache->Drain(&primary_); + } + + void GetStats(AllocatorStatCounters s) const { + stats_.Get(s); + } + + void PrintStats() { + primary_.PrintStats(); + secondary_.PrintStats(); + } + + // ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone + // introspection API. + void ForceLock() { + primary_.ForceLock(); + secondary_.ForceLock(); + } + + void ForceUnlock() { + secondary_.ForceUnlock(); + primary_.ForceUnlock(); + } + + void ReleaseToOS() { primary_.ReleaseToOS(); } + + // Iterate over all existing chunks. + // The allocator must be locked when calling this function. + void ForEachChunk(ForEachChunkCallback callback, void *arg) { + primary_.ForEachChunk(callback, arg); + secondary_.ForEachChunk(callback, arg); + } + + private: + PrimaryAllocator primary_; + SecondaryAllocator secondary_; + AllocatorGlobalStats stats_; + atomic_uint8_t may_return_null_; + atomic_uint8_t rss_limit_is_exceeded_; +}; diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_interface.h b/libsanitizer/sanitizer_common/sanitizer_allocator_interface.h index fd5bed30c4b3..166f8c220492 100644 --- a/libsanitizer/sanitizer_common/sanitizer_allocator_interface.h +++ b/libsanitizer/sanitizer_common/sanitizer_allocator_interface.h @@ -27,10 +27,18 @@ SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_heap_size(); SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_free_bytes(); SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_unmapped_bytes(); +SANITIZER_INTERFACE_ATTRIBUTE int __sanitizer_install_malloc_and_free_hooks( + void (*malloc_hook)(const void *, uptr), + void (*free_hook)(const void *)); + SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE /* OPTIONAL */ void __sanitizer_malloc_hook(void *ptr, uptr size); SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE /* OPTIONAL */ void __sanitizer_free_hook(void *ptr); + + +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE + void __sanitizer_print_memory_profile(int top_percent); } // extern "C" #endif // SANITIZER_ALLOCATOR_INTERFACE_H diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_internal.h b/libsanitizer/sanitizer_common/sanitizer_allocator_internal.h index 99d8516d8cf2..6a8da8f3584a 100644 --- a/libsanitizer/sanitizer_common/sanitizer_allocator_internal.h +++ b/libsanitizer/sanitizer_common/sanitizer_allocator_internal.h @@ -43,7 +43,12 @@ typedef SizeClassAllocatorLocalCache<PrimaryInternalAllocator> typedef CombinedAllocator<PrimaryInternalAllocator, InternalAllocatorCache, LargeMmapAllocator<> > InternalAllocator; -void *InternalAlloc(uptr size, InternalAllocatorCache *cache = nullptr); +void *InternalAlloc(uptr size, InternalAllocatorCache *cache = nullptr, + uptr alignment = 0); +void *InternalRealloc(void *p, uptr size, + InternalAllocatorCache *cache = nullptr); +void *InternalCalloc(uptr countr, uptr size, + InternalAllocatorCache *cache = nullptr); void InternalFree(void *p, InternalAllocatorCache *cache = nullptr); InternalAllocator *internal_allocator(); @@ -54,8 +59,8 @@ enum InternalAllocEnum { } // namespace __sanitizer inline void *operator new(__sanitizer::operator_new_size_type size, - InternalAllocEnum) { - return InternalAlloc(size); + __sanitizer::InternalAllocEnum) { + return __sanitizer::InternalAlloc(size); } #endif // SANITIZER_ALLOCATOR_INTERNAL_H diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_local_cache.h b/libsanitizer/sanitizer_common/sanitizer_allocator_local_cache.h new file mode 100644 index 000000000000..40ef0781adbb --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_allocator_local_cache.h @@ -0,0 +1,246 @@ +//===-- sanitizer_allocator_local_cache.h -----------------------*- C++ -*-===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Part of the Sanitizer Allocator. +// +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_ALLOCATOR_H +#error This file must be included inside sanitizer_allocator.h +#endif + +// Objects of this type should be used as local caches for SizeClassAllocator64 +// or SizeClassAllocator32. Since the typical use of this class is to have one +// object per thread in TLS, is has to be POD. +template<class SizeClassAllocator> +struct SizeClassAllocatorLocalCache + : SizeClassAllocator::AllocatorCache { +}; + +// Cache used by SizeClassAllocator64. +template <class SizeClassAllocator> +struct SizeClassAllocator64LocalCache { + typedef SizeClassAllocator Allocator; + static const uptr kNumClasses = SizeClassAllocator::kNumClasses; + typedef typename Allocator::SizeClassMapT SizeClassMap; + typedef typename Allocator::CompactPtrT CompactPtrT; + + void Init(AllocatorGlobalStats *s) { + stats_.Init(); + if (s) + s->Register(&stats_); + } + + void Destroy(SizeClassAllocator *allocator, AllocatorGlobalStats *s) { + Drain(allocator); + if (s) + s->Unregister(&stats_); + } + + void *Allocate(SizeClassAllocator *allocator, uptr class_id) { + CHECK_NE(class_id, 0UL); + CHECK_LT(class_id, kNumClasses); + stats_.Add(AllocatorStatAllocated, Allocator::ClassIdToSize(class_id)); + PerClass *c = &per_class_[class_id]; + if (UNLIKELY(c->count == 0)) + Refill(c, allocator, class_id); + CHECK_GT(c->count, 0); + CompactPtrT chunk = c->chunks[--c->count]; + void *res = reinterpret_cast<void *>(allocator->CompactPtrToPointer( + allocator->GetRegionBeginBySizeClass(class_id), chunk)); + return res; + } + + void Deallocate(SizeClassAllocator *allocator, uptr class_id, void *p) { + CHECK_NE(class_id, 0UL); + CHECK_LT(class_id, kNumClasses); + // If the first allocator call on a new thread is a deallocation, then + // max_count will be zero, leading to check failure. + InitCache(); + stats_.Sub(AllocatorStatAllocated, Allocator::ClassIdToSize(class_id)); + PerClass *c = &per_class_[class_id]; + CHECK_NE(c->max_count, 0UL); + if (UNLIKELY(c->count == c->max_count)) + Drain(c, allocator, class_id, c->max_count / 2); + CompactPtrT chunk = allocator->PointerToCompactPtr( + allocator->GetRegionBeginBySizeClass(class_id), + reinterpret_cast<uptr>(p)); + c->chunks[c->count++] = chunk; + } + + void Drain(SizeClassAllocator *allocator) { + for (uptr class_id = 0; class_id < kNumClasses; class_id++) { + PerClass *c = &per_class_[class_id]; + while (c->count > 0) + Drain(c, allocator, class_id, c->count); + } + } + + // private: + struct PerClass { + u32 count; + u32 max_count; + CompactPtrT chunks[2 * SizeClassMap::kMaxNumCachedHint]; + }; + PerClass per_class_[kNumClasses]; + AllocatorStats stats_; + + void InitCache() { + if (per_class_[1].max_count) + return; + for (uptr i = 0; i < kNumClasses; i++) { + PerClass *c = &per_class_[i]; + c->max_count = 2 * SizeClassMap::MaxCachedHint(i); + } + } + + NOINLINE void Refill(PerClass *c, SizeClassAllocator *allocator, + uptr class_id) { + InitCache(); + uptr num_requested_chunks = SizeClassMap::MaxCachedHint(class_id); + allocator->GetFromAllocator(&stats_, class_id, c->chunks, + num_requested_chunks); + c->count = num_requested_chunks; + } + + NOINLINE void Drain(PerClass *c, SizeClassAllocator *allocator, uptr class_id, + uptr count) { + InitCache(); + CHECK_GE(c->count, count); + uptr first_idx_to_drain = c->count - count; + c->count -= count; + allocator->ReturnToAllocator(&stats_, class_id, + &c->chunks[first_idx_to_drain], count); + } +}; + +// Cache used by SizeClassAllocator32. +template <class SizeClassAllocator> +struct SizeClassAllocator32LocalCache { + typedef SizeClassAllocator Allocator; + typedef typename Allocator::TransferBatch TransferBatch; + static const uptr kNumClasses = SizeClassAllocator::kNumClasses; + + void Init(AllocatorGlobalStats *s) { + stats_.Init(); + if (s) + s->Register(&stats_); + } + + void Destroy(SizeClassAllocator *allocator, AllocatorGlobalStats *s) { + Drain(allocator); + if (s) + s->Unregister(&stats_); + } + + void *Allocate(SizeClassAllocator *allocator, uptr class_id) { + CHECK_NE(class_id, 0UL); + CHECK_LT(class_id, kNumClasses); + stats_.Add(AllocatorStatAllocated, Allocator::ClassIdToSize(class_id)); + PerClass *c = &per_class_[class_id]; + if (UNLIKELY(c->count == 0)) + Refill(allocator, class_id); + void *res = c->batch[--c->count]; + PREFETCH(c->batch[c->count - 1]); + return res; + } + + void Deallocate(SizeClassAllocator *allocator, uptr class_id, void *p) { + CHECK_NE(class_id, 0UL); + CHECK_LT(class_id, kNumClasses); + // If the first allocator call on a new thread is a deallocation, then + // max_count will be zero, leading to check failure. + InitCache(); + stats_.Sub(AllocatorStatAllocated, Allocator::ClassIdToSize(class_id)); + PerClass *c = &per_class_[class_id]; + CHECK_NE(c->max_count, 0UL); + if (UNLIKELY(c->count == c->max_count)) + Drain(allocator, class_id); + c->batch[c->count++] = p; + } + + void Drain(SizeClassAllocator *allocator) { + for (uptr class_id = 0; class_id < kNumClasses; class_id++) { + PerClass *c = &per_class_[class_id]; + while (c->count > 0) + Drain(allocator, class_id); + } + } + + // private: + typedef typename SizeClassAllocator::SizeClassMapT SizeClassMap; + struct PerClass { + uptr count; + uptr max_count; + void *batch[2 * TransferBatch::kMaxNumCached]; + }; + PerClass per_class_[kNumClasses]; + AllocatorStats stats_; + + void InitCache() { + if (per_class_[1].max_count) + return; + for (uptr i = 0; i < kNumClasses; i++) { + PerClass *c = &per_class_[i]; + c->max_count = 2 * TransferBatch::MaxCached(i); + } + } + + // TransferBatch class is declared in SizeClassAllocator. + // We transfer chunks between central and thread-local free lists in batches. + // For small size classes we allocate batches separately. + // For large size classes we may use one of the chunks to store the batch. + // sizeof(TransferBatch) must be a power of 2 for more efficient allocation. + static uptr SizeClassForTransferBatch(uptr class_id) { + if (Allocator::ClassIdToSize(class_id) < + TransferBatch::AllocationSizeRequiredForNElements( + TransferBatch::MaxCached(class_id))) + return SizeClassMap::ClassID(sizeof(TransferBatch)); + return 0; + } + + // Returns a TransferBatch suitable for class_id. + // For small size classes allocates the batch from the allocator. + // For large size classes simply returns b. + TransferBatch *CreateBatch(uptr class_id, SizeClassAllocator *allocator, + TransferBatch *b) { + if (uptr batch_class_id = SizeClassForTransferBatch(class_id)) + return (TransferBatch*)Allocate(allocator, batch_class_id); + return b; + } + + // Destroys TransferBatch b. + // For small size classes deallocates b to the allocator. + // Does notthing for large size classes. + void DestroyBatch(uptr class_id, SizeClassAllocator *allocator, + TransferBatch *b) { + if (uptr batch_class_id = SizeClassForTransferBatch(class_id)) + Deallocate(allocator, batch_class_id, b); + } + + NOINLINE void Refill(SizeClassAllocator *allocator, uptr class_id) { + InitCache(); + PerClass *c = &per_class_[class_id]; + TransferBatch *b = allocator->AllocateBatch(&stats_, this, class_id); + CHECK_GT(b->Count(), 0); + b->CopyToArray(c->batch); + c->count = b->Count(); + DestroyBatch(class_id, allocator, b); + } + + NOINLINE void Drain(SizeClassAllocator *allocator, uptr class_id) { + InitCache(); + PerClass *c = &per_class_[class_id]; + uptr cnt = Min(c->max_count / 2, c->count); + uptr first_idx_to_drain = c->count - cnt; + TransferBatch *b = CreateBatch( + class_id, allocator, (TransferBatch *)c->batch[first_idx_to_drain]); + b->SetFromArray(allocator->GetRegionBeginBySizeClass(class_id), + &c->batch[first_idx_to_drain], cnt); + c->count -= cnt; + allocator->DeallocateBatch(&stats_, class_id, b); + } +}; diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h b/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h new file mode 100644 index 000000000000..989b87a19234 --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h @@ -0,0 +1,302 @@ +//===-- sanitizer_allocator_primary32.h -------------------------*- C++ -*-===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Part of the Sanitizer Allocator. +// +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_ALLOCATOR_H +#error This file must be included inside sanitizer_allocator.h +#endif + +template<class SizeClassAllocator> struct SizeClassAllocator32LocalCache; + +// SizeClassAllocator32 -- allocator for 32-bit address space. +// This allocator can theoretically be used on 64-bit arch, but there it is less +// efficient than SizeClassAllocator64. +// +// [kSpaceBeg, kSpaceBeg + kSpaceSize) is the range of addresses which can +// be returned by MmapOrDie(). +// +// Region: +// a result of a single call to MmapAlignedOrDie(kRegionSize, kRegionSize). +// Since the regions are aligned by kRegionSize, there are exactly +// kNumPossibleRegions possible regions in the address space and so we keep +// a ByteMap possible_regions to store the size classes of each Region. +// 0 size class means the region is not used by the allocator. +// +// One Region is used to allocate chunks of a single size class. +// A Region looks like this: +// UserChunk1 .. UserChunkN <gap> MetaChunkN .. MetaChunk1 +// +// In order to avoid false sharing the objects of this class should be +// chache-line aligned. +template <const uptr kSpaceBeg, const u64 kSpaceSize, + const uptr kMetadataSize, class SizeClassMap, + const uptr kRegionSizeLog, + class ByteMap, + class MapUnmapCallback = NoOpMapUnmapCallback> +class SizeClassAllocator32 { + public: + struct TransferBatch { + static const uptr kMaxNumCached = SizeClassMap::kMaxNumCachedHint - 2; + void SetFromArray(uptr region_beg_unused, void *batch[], uptr count) { + count_ = count; + CHECK_LE(count_, kMaxNumCached); + for (uptr i = 0; i < count; i++) + batch_[i] = batch[i]; + } + uptr Count() const { return count_; } + void Clear() { count_ = 0; } + void Add(void *ptr) { + batch_[count_++] = ptr; + CHECK_LE(count_, kMaxNumCached); + } + void CopyToArray(void *to_batch[]) { + for (uptr i = 0, n = Count(); i < n; i++) + to_batch[i] = batch_[i]; + } + + // How much memory do we need for a batch containing n elements. + static uptr AllocationSizeRequiredForNElements(uptr n) { + return sizeof(uptr) * 2 + sizeof(void *) * n; + } + static uptr MaxCached(uptr class_id) { + return Min(kMaxNumCached, SizeClassMap::MaxCachedHint(class_id)); + } + + TransferBatch *next; + + private: + uptr count_; + void *batch_[kMaxNumCached]; + }; + + static const uptr kBatchSize = sizeof(TransferBatch); + COMPILER_CHECK((kBatchSize & (kBatchSize - 1)) == 0); + COMPILER_CHECK(sizeof(TransferBatch) == + SizeClassMap::kMaxNumCachedHint * sizeof(uptr)); + + static uptr ClassIdToSize(uptr class_id) { + return SizeClassMap::Size(class_id); + } + + typedef SizeClassAllocator32<kSpaceBeg, kSpaceSize, kMetadataSize, + SizeClassMap, kRegionSizeLog, ByteMap, MapUnmapCallback> ThisT; + typedef SizeClassAllocator32LocalCache<ThisT> AllocatorCache; + + void Init() { + possible_regions.TestOnlyInit(); + internal_memset(size_class_info_array, 0, sizeof(size_class_info_array)); + } + + void *MapWithCallback(uptr size) { + size = RoundUpTo(size, GetPageSizeCached()); + void *res = MmapOrDie(size, "SizeClassAllocator32"); + MapUnmapCallback().OnMap((uptr)res, size); + return res; + } + + void UnmapWithCallback(uptr beg, uptr size) { + MapUnmapCallback().OnUnmap(beg, size); + UnmapOrDie(reinterpret_cast<void *>(beg), size); + } + + static bool CanAllocate(uptr size, uptr alignment) { + return size <= SizeClassMap::kMaxSize && + alignment <= SizeClassMap::kMaxSize; + } + + void *GetMetaData(const void *p) { + CHECK(PointerIsMine(p)); + uptr mem = reinterpret_cast<uptr>(p); + uptr beg = ComputeRegionBeg(mem); + uptr size = ClassIdToSize(GetSizeClass(p)); + u32 offset = mem - beg; + uptr n = offset / (u32)size; // 32-bit division + uptr meta = (beg + kRegionSize) - (n + 1) * kMetadataSize; + return reinterpret_cast<void*>(meta); + } + + NOINLINE TransferBatch *AllocateBatch(AllocatorStats *stat, AllocatorCache *c, + uptr class_id) { + CHECK_LT(class_id, kNumClasses); + SizeClassInfo *sci = GetSizeClassInfo(class_id); + SpinMutexLock l(&sci->mutex); + if (sci->free_list.empty()) + PopulateFreeList(stat, c, sci, class_id); + CHECK(!sci->free_list.empty()); + TransferBatch *b = sci->free_list.front(); + sci->free_list.pop_front(); + return b; + } + + NOINLINE void DeallocateBatch(AllocatorStats *stat, uptr class_id, + TransferBatch *b) { + CHECK_LT(class_id, kNumClasses); + SizeClassInfo *sci = GetSizeClassInfo(class_id); + SpinMutexLock l(&sci->mutex); + CHECK_GT(b->Count(), 0); + sci->free_list.push_front(b); + } + + uptr GetRegionBeginBySizeClass(uptr class_id) { return 0; } + + bool PointerIsMine(const void *p) { + uptr mem = reinterpret_cast<uptr>(p); + if (mem < kSpaceBeg || mem >= kSpaceBeg + kSpaceSize) + return false; + return GetSizeClass(p) != 0; + } + + uptr GetSizeClass(const void *p) { + return possible_regions[ComputeRegionId(reinterpret_cast<uptr>(p))]; + } + + void *GetBlockBegin(const void *p) { + CHECK(PointerIsMine(p)); + uptr mem = reinterpret_cast<uptr>(p); + uptr beg = ComputeRegionBeg(mem); + uptr size = ClassIdToSize(GetSizeClass(p)); + u32 offset = mem - beg; + u32 n = offset / (u32)size; // 32-bit division + uptr res = beg + (n * (u32)size); + return reinterpret_cast<void*>(res); + } + + uptr GetActuallyAllocatedSize(void *p) { + CHECK(PointerIsMine(p)); + return ClassIdToSize(GetSizeClass(p)); + } + + uptr ClassID(uptr size) { return SizeClassMap::ClassID(size); } + + uptr TotalMemoryUsed() { + // No need to lock here. + uptr res = 0; + for (uptr i = 0; i < kNumPossibleRegions; i++) + if (possible_regions[i]) + res += kRegionSize; + return res; + } + + void TestOnlyUnmap() { + for (uptr i = 0; i < kNumPossibleRegions; i++) + if (possible_regions[i]) + UnmapWithCallback((i * kRegionSize), kRegionSize); + } + + // ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone + // introspection API. + void ForceLock() { + for (uptr i = 0; i < kNumClasses; i++) { + GetSizeClassInfo(i)->mutex.Lock(); + } + } + + void ForceUnlock() { + for (int i = kNumClasses - 1; i >= 0; i--) { + GetSizeClassInfo(i)->mutex.Unlock(); + } + } + + // Iterate over all existing chunks. + // The allocator must be locked when calling this function. + void ForEachChunk(ForEachChunkCallback callback, void *arg) { + for (uptr region = 0; region < kNumPossibleRegions; region++) + if (possible_regions[region]) { + uptr chunk_size = ClassIdToSize(possible_regions[region]); + uptr max_chunks_in_region = kRegionSize / (chunk_size + kMetadataSize); + uptr region_beg = region * kRegionSize; + for (uptr chunk = region_beg; + chunk < region_beg + max_chunks_in_region * chunk_size; + chunk += chunk_size) { + // Too slow: CHECK_EQ((void *)chunk, GetBlockBegin((void *)chunk)); + callback(chunk, arg); + } + } + } + + void PrintStats() { + } + + static uptr AdditionalSize() { + return 0; + } + + // This is empty here. Currently only implemented in 64-bit allocator. + void ReleaseToOS() { } + + + typedef SizeClassMap SizeClassMapT; + static const uptr kNumClasses = SizeClassMap::kNumClasses; + + private: + static const uptr kRegionSize = 1 << kRegionSizeLog; + static const uptr kNumPossibleRegions = kSpaceSize / kRegionSize; + + struct SizeClassInfo { + SpinMutex mutex; + IntrusiveList<TransferBatch> free_list; + char padding[kCacheLineSize - sizeof(uptr) - + sizeof(IntrusiveList<TransferBatch>)]; + }; + COMPILER_CHECK(sizeof(SizeClassInfo) == kCacheLineSize); + + uptr ComputeRegionId(uptr mem) { + uptr res = mem >> kRegionSizeLog; + CHECK_LT(res, kNumPossibleRegions); + return res; + } + + uptr ComputeRegionBeg(uptr mem) { + return mem & ~(kRegionSize - 1); + } + + uptr AllocateRegion(AllocatorStats *stat, uptr class_id) { + CHECK_LT(class_id, kNumClasses); + uptr res = reinterpret_cast<uptr>(MmapAlignedOrDie(kRegionSize, kRegionSize, + "SizeClassAllocator32")); + MapUnmapCallback().OnMap(res, kRegionSize); + stat->Add(AllocatorStatMapped, kRegionSize); + CHECK_EQ(0U, (res & (kRegionSize - 1))); + possible_regions.set(ComputeRegionId(res), static_cast<u8>(class_id)); + return res; + } + + SizeClassInfo *GetSizeClassInfo(uptr class_id) { + CHECK_LT(class_id, kNumClasses); + return &size_class_info_array[class_id]; + } + + void PopulateFreeList(AllocatorStats *stat, AllocatorCache *c, + SizeClassInfo *sci, uptr class_id) { + uptr size = ClassIdToSize(class_id); + uptr reg = AllocateRegion(stat, class_id); + uptr n_chunks = kRegionSize / (size + kMetadataSize); + uptr max_count = TransferBatch::MaxCached(class_id); + TransferBatch *b = nullptr; + for (uptr i = reg; i < reg + n_chunks * size; i += size) { + if (!b) { + b = c->CreateBatch(class_id, this, (TransferBatch*)i); + b->Clear(); + } + b->Add((void*)i); + if (b->Count() == max_count) { + CHECK_GT(b->Count(), 0); + sci->free_list.push_back(b); + b = nullptr; + } + } + if (b) { + CHECK_GT(b->Count(), 0); + sci->free_list.push_back(b); + } + } + + ByteMap possible_regions; + SizeClassInfo size_class_info_array[kNumClasses]; +}; diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h b/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h new file mode 100644 index 000000000000..620639afbfdb --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h @@ -0,0 +1,503 @@ +//===-- sanitizer_allocator_primary64.h -------------------------*- C++ -*-===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Part of the Sanitizer Allocator. +// +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_ALLOCATOR_H +#error This file must be included inside sanitizer_allocator.h +#endif + +template<class SizeClassAllocator> struct SizeClassAllocator64LocalCache; + +// SizeClassAllocator64 -- allocator for 64-bit address space. +// The template parameter Params is a class containing the actual parameters. +// +// Space: a portion of address space of kSpaceSize bytes starting at SpaceBeg. +// If kSpaceBeg is ~0 then SpaceBeg is chosen dynamically my mmap. +// Otherwise SpaceBeg=kSpaceBeg (fixed address). +// kSpaceSize is a power of two. +// At the beginning the entire space is mprotect-ed, then small parts of it +// are mapped on demand. +// +// Region: a part of Space dedicated to a single size class. +// There are kNumClasses Regions of equal size. +// +// UserChunk: a piece of memory returned to user. +// MetaChunk: kMetadataSize bytes of metadata associated with a UserChunk. + +// FreeArray is an array free-d chunks (stored as 4-byte offsets) +// +// A Region looks like this: +// UserChunk1 ... UserChunkN <gap> MetaChunkN ... MetaChunk1 FreeArray + +struct SizeClassAllocator64FlagMasks { // Bit masks. + enum { + kRandomShuffleChunks = 1, + }; +}; + +template <class Params> +class SizeClassAllocator64 { + public: + static const uptr kSpaceBeg = Params::kSpaceBeg; + static const uptr kSpaceSize = Params::kSpaceSize; + static const uptr kMetadataSize = Params::kMetadataSize; + typedef typename Params::SizeClassMap SizeClassMap; + typedef typename Params::MapUnmapCallback MapUnmapCallback; + + static const bool kRandomShuffleChunks = + Params::kFlags & SizeClassAllocator64FlagMasks::kRandomShuffleChunks; + + typedef SizeClassAllocator64<Params> ThisT; + typedef SizeClassAllocator64LocalCache<ThisT> AllocatorCache; + + // When we know the size class (the region base) we can represent a pointer + // as a 4-byte integer (offset from the region start shifted right by 4). + typedef u32 CompactPtrT; + static const uptr kCompactPtrScale = 4; + CompactPtrT PointerToCompactPtr(uptr base, uptr ptr) { + return static_cast<CompactPtrT>((ptr - base) >> kCompactPtrScale); + } + uptr CompactPtrToPointer(uptr base, CompactPtrT ptr32) { + return base + (static_cast<uptr>(ptr32) << kCompactPtrScale); + } + + void Init() { + uptr TotalSpaceSize = kSpaceSize + AdditionalSize(); + if (kUsingConstantSpaceBeg) { + CHECK_EQ(kSpaceBeg, reinterpret_cast<uptr>( + MmapFixedNoAccess(kSpaceBeg, TotalSpaceSize))); + } else { + NonConstSpaceBeg = + reinterpret_cast<uptr>(MmapNoAccess(TotalSpaceSize)); + CHECK_NE(NonConstSpaceBeg, ~(uptr)0); + } + MapWithCallback(SpaceEnd(), AdditionalSize()); + } + + void MapWithCallback(uptr beg, uptr size) { + CHECK_EQ(beg, reinterpret_cast<uptr>(MmapFixedOrDie(beg, size))); + MapUnmapCallback().OnMap(beg, size); + } + + void UnmapWithCallback(uptr beg, uptr size) { + MapUnmapCallback().OnUnmap(beg, size); + UnmapOrDie(reinterpret_cast<void *>(beg), size); + } + + static bool CanAllocate(uptr size, uptr alignment) { + return size <= SizeClassMap::kMaxSize && + alignment <= SizeClassMap::kMaxSize; + } + + NOINLINE void ReturnToAllocator(AllocatorStats *stat, uptr class_id, + const CompactPtrT *chunks, uptr n_chunks) { + RegionInfo *region = GetRegionInfo(class_id); + uptr region_beg = GetRegionBeginBySizeClass(class_id); + CompactPtrT *free_array = GetFreeArray(region_beg); + + BlockingMutexLock l(®ion->mutex); + uptr old_num_chunks = region->num_freed_chunks; + uptr new_num_freed_chunks = old_num_chunks + n_chunks; + EnsureFreeArraySpace(region, region_beg, new_num_freed_chunks); + for (uptr i = 0; i < n_chunks; i++) + free_array[old_num_chunks + i] = chunks[i]; + region->num_freed_chunks = new_num_freed_chunks; + region->n_freed += n_chunks; + } + + NOINLINE void GetFromAllocator(AllocatorStats *stat, uptr class_id, + CompactPtrT *chunks, uptr n_chunks) { + RegionInfo *region = GetRegionInfo(class_id); + uptr region_beg = GetRegionBeginBySizeClass(class_id); + CompactPtrT *free_array = GetFreeArray(region_beg); + + BlockingMutexLock l(®ion->mutex); + if (UNLIKELY(region->num_freed_chunks < n_chunks)) { + PopulateFreeArray(stat, class_id, region, + n_chunks - region->num_freed_chunks); + CHECK_GE(region->num_freed_chunks, n_chunks); + } + region->num_freed_chunks -= n_chunks; + uptr base_idx = region->num_freed_chunks; + for (uptr i = 0; i < n_chunks; i++) + chunks[i] = free_array[base_idx + i]; + region->n_allocated += n_chunks; + } + + + bool PointerIsMine(const void *p) { + uptr P = reinterpret_cast<uptr>(p); + if (kUsingConstantSpaceBeg && (kSpaceBeg % kSpaceSize) == 0) + return P / kSpaceSize == kSpaceBeg / kSpaceSize; + return P >= SpaceBeg() && P < SpaceEnd(); + } + + uptr GetRegionBegin(const void *p) { + if (kUsingConstantSpaceBeg) + return reinterpret_cast<uptr>(p) & ~(kRegionSize - 1); + uptr space_beg = SpaceBeg(); + return ((reinterpret_cast<uptr>(p) - space_beg) & ~(kRegionSize - 1)) + + space_beg; + } + + uptr GetRegionBeginBySizeClass(uptr class_id) { + return SpaceBeg() + kRegionSize * class_id; + } + + uptr GetSizeClass(const void *p) { + if (kUsingConstantSpaceBeg && (kSpaceBeg % kSpaceSize) == 0) + return ((reinterpret_cast<uptr>(p)) / kRegionSize) % kNumClassesRounded; + return ((reinterpret_cast<uptr>(p) - SpaceBeg()) / kRegionSize) % + kNumClassesRounded; + } + + void *GetBlockBegin(const void *p) { + uptr class_id = GetSizeClass(p); + uptr size = ClassIdToSize(class_id); + if (!size) return nullptr; + uptr chunk_idx = GetChunkIdx((uptr)p, size); + uptr reg_beg = GetRegionBegin(p); + uptr beg = chunk_idx * size; + uptr next_beg = beg + size; + if (class_id >= kNumClasses) return nullptr; + RegionInfo *region = GetRegionInfo(class_id); + if (region->mapped_user >= next_beg) + return reinterpret_cast<void*>(reg_beg + beg); + return nullptr; + } + + uptr GetActuallyAllocatedSize(void *p) { + CHECK(PointerIsMine(p)); + return ClassIdToSize(GetSizeClass(p)); + } + + uptr ClassID(uptr size) { return SizeClassMap::ClassID(size); } + + void *GetMetaData(const void *p) { + uptr class_id = GetSizeClass(p); + uptr size = ClassIdToSize(class_id); + uptr chunk_idx = GetChunkIdx(reinterpret_cast<uptr>(p), size); + uptr region_beg = GetRegionBeginBySizeClass(class_id); + return reinterpret_cast<void *>(GetMetadataEnd(region_beg) - + (1 + chunk_idx) * kMetadataSize); + } + + uptr TotalMemoryUsed() { + uptr res = 0; + for (uptr i = 0; i < kNumClasses; i++) + res += GetRegionInfo(i)->allocated_user; + return res; + } + + // Test-only. + void TestOnlyUnmap() { + UnmapWithCallback(SpaceBeg(), kSpaceSize + AdditionalSize()); + } + + static void FillMemoryProfile(uptr start, uptr rss, bool file, uptr *stats, + uptr stats_size) { + for (uptr class_id = 0; class_id < stats_size; class_id++) + if (stats[class_id] == start) + stats[class_id] = rss; + } + + void PrintStats(uptr class_id, uptr rss) { + RegionInfo *region = GetRegionInfo(class_id); + if (region->mapped_user == 0) return; + uptr in_use = region->n_allocated - region->n_freed; + uptr avail_chunks = region->allocated_user / ClassIdToSize(class_id); + Printf( + " %02zd (%zd): mapped: %zdK allocs: %zd frees: %zd inuse: %zd " + "num_freed_chunks %zd" + " avail: %zd rss: %zdK releases: %zd\n", + class_id, ClassIdToSize(class_id), region->mapped_user >> 10, + region->n_allocated, region->n_freed, in_use, + region->num_freed_chunks, avail_chunks, rss >> 10, + region->rtoi.num_releases); + } + + void PrintStats() { + uptr total_mapped = 0; + uptr n_allocated = 0; + uptr n_freed = 0; + for (uptr class_id = 1; class_id < kNumClasses; class_id++) { + RegionInfo *region = GetRegionInfo(class_id); + total_mapped += region->mapped_user; + n_allocated += region->n_allocated; + n_freed += region->n_freed; + } + Printf("Stats: SizeClassAllocator64: %zdM mapped in %zd allocations; " + "remains %zd\n", + total_mapped >> 20, n_allocated, n_allocated - n_freed); + uptr rss_stats[kNumClasses]; + for (uptr class_id = 0; class_id < kNumClasses; class_id++) + rss_stats[class_id] = SpaceBeg() + kRegionSize * class_id; + GetMemoryProfile(FillMemoryProfile, rss_stats, kNumClasses); + for (uptr class_id = 1; class_id < kNumClasses; class_id++) + PrintStats(class_id, rss_stats[class_id]); + } + + // ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone + // introspection API. + void ForceLock() { + for (uptr i = 0; i < kNumClasses; i++) { + GetRegionInfo(i)->mutex.Lock(); + } + } + + void ForceUnlock() { + for (int i = (int)kNumClasses - 1; i >= 0; i--) { + GetRegionInfo(i)->mutex.Unlock(); + } + } + + // Iterate over all existing chunks. + // The allocator must be locked when calling this function. + void ForEachChunk(ForEachChunkCallback callback, void *arg) { + for (uptr class_id = 1; class_id < kNumClasses; class_id++) { + RegionInfo *region = GetRegionInfo(class_id); + uptr chunk_size = ClassIdToSize(class_id); + uptr region_beg = SpaceBeg() + class_id * kRegionSize; + for (uptr chunk = region_beg; + chunk < region_beg + region->allocated_user; + chunk += chunk_size) { + // Too slow: CHECK_EQ((void *)chunk, GetBlockBegin((void *)chunk)); + callback(chunk, arg); + } + } + } + + static uptr ClassIdToSize(uptr class_id) { + return SizeClassMap::Size(class_id); + } + + static uptr AdditionalSize() { + return RoundUpTo(sizeof(RegionInfo) * kNumClassesRounded, + GetPageSizeCached()); + } + + void ReleaseToOS() { + for (uptr class_id = 1; class_id < kNumClasses; class_id++) + ReleaseToOS(class_id); + } + + typedef SizeClassMap SizeClassMapT; + static const uptr kNumClasses = SizeClassMap::kNumClasses; + static const uptr kNumClassesRounded = SizeClassMap::kNumClassesRounded; + + private: + static const uptr kRegionSize = kSpaceSize / kNumClassesRounded; + // FreeArray is the array of free-d chunks (stored as 4-byte offsets). + // In the worst case it may reguire kRegionSize/SizeClassMap::kMinSize + // elements, but in reality this will not happen. For simplicity we + // dedicate 1/8 of the region's virtual space to FreeArray. + static const uptr kFreeArraySize = kRegionSize / 8; + + static const bool kUsingConstantSpaceBeg = kSpaceBeg != ~(uptr)0; + uptr NonConstSpaceBeg; + uptr SpaceBeg() const { + return kUsingConstantSpaceBeg ? kSpaceBeg : NonConstSpaceBeg; + } + uptr SpaceEnd() const { return SpaceBeg() + kSpaceSize; } + // kRegionSize must be >= 2^32. + COMPILER_CHECK((kRegionSize) >= (1ULL << (SANITIZER_WORDSIZE / 2))); + // kRegionSize must be <= 2^36, see CompactPtrT. + COMPILER_CHECK((kRegionSize) <= (1ULL << (SANITIZER_WORDSIZE / 2 + 4))); + // Call mmap for user memory with at least this size. + static const uptr kUserMapSize = 1 << 16; + // Call mmap for metadata memory with at least this size. + static const uptr kMetaMapSize = 1 << 16; + // Call mmap for free array memory with at least this size. + static const uptr kFreeArrayMapSize = 1 << 16; + // Granularity of ReleaseToOs (aka madvise). + static const uptr kReleaseToOsGranularity = 1 << 12; + + struct ReleaseToOsInfo { + uptr n_freed_at_last_release; + uptr num_releases; + }; + + struct RegionInfo { + BlockingMutex mutex; + uptr num_freed_chunks; // Number of elements in the freearray. + uptr mapped_free_array; // Bytes mapped for freearray. + uptr allocated_user; // Bytes allocated for user memory. + uptr allocated_meta; // Bytes allocated for metadata. + uptr mapped_user; // Bytes mapped for user memory. + uptr mapped_meta; // Bytes mapped for metadata. + u32 rand_state; // Seed for random shuffle, used if kRandomShuffleChunks. + uptr n_allocated, n_freed; // Just stats. + ReleaseToOsInfo rtoi; + }; + COMPILER_CHECK(sizeof(RegionInfo) >= kCacheLineSize); + + u32 Rand(u32 *state) { // ANSI C linear congruential PRNG. + return (*state = *state * 1103515245 + 12345) >> 16; + } + + u32 RandN(u32 *state, u32 n) { return Rand(state) % n; } // [0, n) + + void RandomShuffle(u32 *a, u32 n, u32 *rand_state) { + if (n <= 1) return; + for (u32 i = n - 1; i > 0; i--) + Swap(a[i], a[RandN(rand_state, i + 1)]); + } + + RegionInfo *GetRegionInfo(uptr class_id) { + CHECK_LT(class_id, kNumClasses); + RegionInfo *regions = + reinterpret_cast<RegionInfo *>(SpaceBeg() + kSpaceSize); + return ®ions[class_id]; + } + + uptr GetMetadataEnd(uptr region_beg) { + return region_beg + kRegionSize - kFreeArraySize; + } + + uptr GetChunkIdx(uptr chunk, uptr size) { + if (!kUsingConstantSpaceBeg) + chunk -= SpaceBeg(); + + uptr offset = chunk % kRegionSize; + // Here we divide by a non-constant. This is costly. + // size always fits into 32-bits. If the offset fits too, use 32-bit div. + if (offset >> (SANITIZER_WORDSIZE / 2)) + return offset / size; + return (u32)offset / (u32)size; + } + + CompactPtrT *GetFreeArray(uptr region_beg) { + return reinterpret_cast<CompactPtrT *>(region_beg + kRegionSize - + kFreeArraySize); + } + + void EnsureFreeArraySpace(RegionInfo *region, uptr region_beg, + uptr num_freed_chunks) { + uptr needed_space = num_freed_chunks * sizeof(CompactPtrT); + if (region->mapped_free_array < needed_space) { + CHECK_LE(needed_space, kFreeArraySize); + uptr new_mapped_free_array = RoundUpTo(needed_space, kFreeArrayMapSize); + uptr current_map_end = reinterpret_cast<uptr>(GetFreeArray(region_beg)) + + region->mapped_free_array; + uptr new_map_size = new_mapped_free_array - region->mapped_free_array; + MapWithCallback(current_map_end, new_map_size); + region->mapped_free_array = new_mapped_free_array; + } + } + + + NOINLINE void PopulateFreeArray(AllocatorStats *stat, uptr class_id, + RegionInfo *region, uptr requested_count) { + // region->mutex is held. + uptr size = ClassIdToSize(class_id); + uptr beg_idx = region->allocated_user; + uptr end_idx = beg_idx + requested_count * size; + uptr region_beg = GetRegionBeginBySizeClass(class_id); + if (end_idx > region->mapped_user) { + if (!kUsingConstantSpaceBeg && region->mapped_user == 0) + region->rand_state = static_cast<u32>(region_beg >> 12); // From ASLR. + // Do the mmap for the user memory. + uptr map_size = kUserMapSize; + while (end_idx > region->mapped_user + map_size) + map_size += kUserMapSize; + CHECK_GE(region->mapped_user + map_size, end_idx); + MapWithCallback(region_beg + region->mapped_user, map_size); + stat->Add(AllocatorStatMapped, map_size); + region->mapped_user += map_size; + } + CompactPtrT *free_array = GetFreeArray(region_beg); + uptr total_count = (region->mapped_user - beg_idx) / size; + uptr num_freed_chunks = region->num_freed_chunks; + EnsureFreeArraySpace(region, region_beg, num_freed_chunks + total_count); + for (uptr i = 0; i < total_count; i++) { + uptr chunk = beg_idx + i * size; + free_array[num_freed_chunks + total_count - 1 - i] = + PointerToCompactPtr(0, chunk); + } + if (kRandomShuffleChunks) + RandomShuffle(&free_array[num_freed_chunks], total_count, + ®ion->rand_state); + region->num_freed_chunks += total_count; + region->allocated_user += total_count * size; + CHECK_LE(region->allocated_user, region->mapped_user); + + region->allocated_meta += total_count * kMetadataSize; + if (region->allocated_meta > region->mapped_meta) { + uptr map_size = kMetaMapSize; + while (region->allocated_meta > region->mapped_meta + map_size) + map_size += kMetaMapSize; + // Do the mmap for the metadata. + CHECK_GE(region->mapped_meta + map_size, region->allocated_meta); + MapWithCallback(GetMetadataEnd(region_beg) - + region->mapped_meta - map_size, map_size); + region->mapped_meta += map_size; + } + CHECK_LE(region->allocated_meta, region->mapped_meta); + if (region->mapped_user + region->mapped_meta > + kRegionSize - kFreeArraySize) { + Printf("%s: Out of memory. Dying. ", SanitizerToolName); + Printf("The process has exhausted %zuMB for size class %zu.\n", + kRegionSize / 1024 / 1024, size); + Die(); + } + } + + bool MaybeReleaseChunkRange(uptr region_beg, uptr chunk_size, + CompactPtrT first, CompactPtrT last) { + uptr beg_ptr = CompactPtrToPointer(region_beg, first); + uptr end_ptr = CompactPtrToPointer(region_beg, last) + chunk_size; + CHECK_GE(end_ptr - beg_ptr, kReleaseToOsGranularity); + beg_ptr = RoundUpTo(beg_ptr, kReleaseToOsGranularity); + end_ptr = RoundDownTo(end_ptr, kReleaseToOsGranularity); + if (end_ptr == beg_ptr) return false; + ReleaseMemoryToOS(beg_ptr, end_ptr - beg_ptr); + return true; + } + + // Releases some RAM back to OS. + // Algorithm: + // * Lock the region. + // * Sort the chunks. + // * Find ranges fully covered by free-d chunks + // * Release them to OS with madvise. + // + // TODO(kcc): make sure we don't do it too frequently. + void ReleaseToOS(uptr class_id) { + RegionInfo *region = GetRegionInfo(class_id); + uptr region_beg = GetRegionBeginBySizeClass(class_id); + CompactPtrT *free_array = GetFreeArray(region_beg); + uptr chunk_size = ClassIdToSize(class_id); + uptr scaled_chunk_size = chunk_size >> kCompactPtrScale; + const uptr kScaledGranularity = kReleaseToOsGranularity >> kCompactPtrScale; + BlockingMutexLock l(®ion->mutex); + uptr n = region->num_freed_chunks; + if (n * chunk_size < kReleaseToOsGranularity) + return; // No chance to release anything. + if ((region->rtoi.n_freed_at_last_release - region->n_freed) * chunk_size < + kReleaseToOsGranularity) + return; // Nothing new to release. + SortArray(free_array, n); + uptr beg = free_array[0]; + uptr prev = free_array[0]; + for (uptr i = 1; i < n; i++) { + uptr chunk = free_array[i]; + CHECK_GT(chunk, prev); + if (chunk - prev != scaled_chunk_size) { + CHECK_GT(chunk - prev, scaled_chunk_size); + if (prev + scaled_chunk_size - beg >= kScaledGranularity) { + MaybeReleaseChunkRange(region_beg, chunk_size, beg, prev); + region->rtoi.n_freed_at_last_release = region->n_freed; + region->rtoi.num_releases++; + } + beg = chunk; + } + prev = chunk; + } + } +}; diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h b/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h new file mode 100644 index 000000000000..91c2ecc5b265 --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h @@ -0,0 +1,271 @@ +//===-- sanitizer_allocator_secondary.h -------------------------*- C++ -*-===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Part of the Sanitizer Allocator. +// +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_ALLOCATOR_H +#error This file must be included inside sanitizer_allocator.h +#endif + +// This class can (de)allocate only large chunks of memory using mmap/unmap. +// The main purpose of this allocator is to cover large and rare allocation +// sizes not covered by more efficient allocators (e.g. SizeClassAllocator64). +template <class MapUnmapCallback = NoOpMapUnmapCallback> +class LargeMmapAllocator { + public: + void InitLinkerInitialized(bool may_return_null) { + page_size_ = GetPageSizeCached(); + atomic_store(&may_return_null_, may_return_null, memory_order_relaxed); + } + + void Init(bool may_return_null) { + internal_memset(this, 0, sizeof(*this)); + InitLinkerInitialized(may_return_null); + } + + void *Allocate(AllocatorStats *stat, uptr size, uptr alignment) { + CHECK(IsPowerOfTwo(alignment)); + uptr map_size = RoundUpMapSize(size); + if (alignment > page_size_) + map_size += alignment; + // Overflow. + if (map_size < size) return ReturnNullOrDieOnBadRequest(); + uptr map_beg = reinterpret_cast<uptr>( + MmapOrDie(map_size, "LargeMmapAllocator")); + CHECK(IsAligned(map_beg, page_size_)); + MapUnmapCallback().OnMap(map_beg, map_size); + uptr map_end = map_beg + map_size; + uptr res = map_beg + page_size_; + if (res & (alignment - 1)) // Align. + res += alignment - (res & (alignment - 1)); + CHECK(IsAligned(res, alignment)); + CHECK(IsAligned(res, page_size_)); + CHECK_GE(res + size, map_beg); + CHECK_LE(res + size, map_end); + Header *h = GetHeader(res); + h->size = size; + h->map_beg = map_beg; + h->map_size = map_size; + uptr size_log = MostSignificantSetBitIndex(map_size); + CHECK_LT(size_log, ARRAY_SIZE(stats.by_size_log)); + { + SpinMutexLock l(&mutex_); + uptr idx = n_chunks_++; + chunks_sorted_ = false; + CHECK_LT(idx, kMaxNumChunks); + h->chunk_idx = idx; + chunks_[idx] = h; + stats.n_allocs++; + stats.currently_allocated += map_size; + stats.max_allocated = Max(stats.max_allocated, stats.currently_allocated); + stats.by_size_log[size_log]++; + stat->Add(AllocatorStatAllocated, map_size); + stat->Add(AllocatorStatMapped, map_size); + } + return reinterpret_cast<void*>(res); + } + + bool MayReturnNull() const { + return atomic_load(&may_return_null_, memory_order_acquire); + } + + void *ReturnNullOrDieOnBadRequest() { + if (MayReturnNull()) return nullptr; + ReportAllocatorCannotReturnNull(false); + } + + void *ReturnNullOrDieOnOOM() { + if (MayReturnNull()) return nullptr; + ReportAllocatorCannotReturnNull(true); + } + + void SetMayReturnNull(bool may_return_null) { + atomic_store(&may_return_null_, may_return_null, memory_order_release); + } + + void Deallocate(AllocatorStats *stat, void *p) { + Header *h = GetHeader(p); + { + SpinMutexLock l(&mutex_); + uptr idx = h->chunk_idx; + CHECK_EQ(chunks_[idx], h); + CHECK_LT(idx, n_chunks_); + chunks_[idx] = chunks_[n_chunks_ - 1]; + chunks_[idx]->chunk_idx = idx; + n_chunks_--; + chunks_sorted_ = false; + stats.n_frees++; + stats.currently_allocated -= h->map_size; + stat->Sub(AllocatorStatAllocated, h->map_size); + stat->Sub(AllocatorStatMapped, h->map_size); + } + MapUnmapCallback().OnUnmap(h->map_beg, h->map_size); + UnmapOrDie(reinterpret_cast<void*>(h->map_beg), h->map_size); + } + + uptr TotalMemoryUsed() { + SpinMutexLock l(&mutex_); + uptr res = 0; + for (uptr i = 0; i < n_chunks_; i++) { + Header *h = chunks_[i]; + CHECK_EQ(h->chunk_idx, i); + res += RoundUpMapSize(h->size); + } + return res; + } + + bool PointerIsMine(const void *p) { + return GetBlockBegin(p) != nullptr; + } + + uptr GetActuallyAllocatedSize(void *p) { + return RoundUpTo(GetHeader(p)->size, page_size_); + } + + // At least page_size_/2 metadata bytes is available. + void *GetMetaData(const void *p) { + // Too slow: CHECK_EQ(p, GetBlockBegin(p)); + if (!IsAligned(reinterpret_cast<uptr>(p), page_size_)) { + Printf("%s: bad pointer %p\n", SanitizerToolName, p); + CHECK(IsAligned(reinterpret_cast<uptr>(p), page_size_)); + } + return GetHeader(p) + 1; + } + + void *GetBlockBegin(const void *ptr) { + uptr p = reinterpret_cast<uptr>(ptr); + SpinMutexLock l(&mutex_); + uptr nearest_chunk = 0; + // Cache-friendly linear search. + for (uptr i = 0; i < n_chunks_; i++) { + uptr ch = reinterpret_cast<uptr>(chunks_[i]); + if (p < ch) continue; // p is at left to this chunk, skip it. + if (p - ch < p - nearest_chunk) + nearest_chunk = ch; + } + if (!nearest_chunk) + return nullptr; + Header *h = reinterpret_cast<Header *>(nearest_chunk); + CHECK_GE(nearest_chunk, h->map_beg); + CHECK_LT(nearest_chunk, h->map_beg + h->map_size); + CHECK_LE(nearest_chunk, p); + if (h->map_beg + h->map_size <= p) + return nullptr; + return GetUser(h); + } + + // This function does the same as GetBlockBegin, but is much faster. + // Must be called with the allocator locked. + void *GetBlockBeginFastLocked(void *ptr) { + mutex_.CheckLocked(); + uptr p = reinterpret_cast<uptr>(ptr); + uptr n = n_chunks_; + if (!n) return nullptr; + if (!chunks_sorted_) { + // Do one-time sort. chunks_sorted_ is reset in Allocate/Deallocate. + SortArray(reinterpret_cast<uptr*>(chunks_), n); + for (uptr i = 0; i < n; i++) + chunks_[i]->chunk_idx = i; + chunks_sorted_ = true; + min_mmap_ = reinterpret_cast<uptr>(chunks_[0]); + max_mmap_ = reinterpret_cast<uptr>(chunks_[n - 1]) + + chunks_[n - 1]->map_size; + } + if (p < min_mmap_ || p >= max_mmap_) + return nullptr; + uptr beg = 0, end = n - 1; + // This loop is a log(n) lower_bound. It does not check for the exact match + // to avoid expensive cache-thrashing loads. + while (end - beg >= 2) { + uptr mid = (beg + end) / 2; // Invariant: mid >= beg + 1 + if (p < reinterpret_cast<uptr>(chunks_[mid])) + end = mid - 1; // We are not interested in chunks_[mid]. + else + beg = mid; // chunks_[mid] may still be what we want. + } + + if (beg < end) { + CHECK_EQ(beg + 1, end); + // There are 2 chunks left, choose one. + if (p >= reinterpret_cast<uptr>(chunks_[end])) + beg = end; + } + + Header *h = chunks_[beg]; + if (h->map_beg + h->map_size <= p || p < h->map_beg) + return nullptr; + return GetUser(h); + } + + void PrintStats() { + Printf("Stats: LargeMmapAllocator: allocated %zd times, " + "remains %zd (%zd K) max %zd M; by size logs: ", + stats.n_allocs, stats.n_allocs - stats.n_frees, + stats.currently_allocated >> 10, stats.max_allocated >> 20); + for (uptr i = 0; i < ARRAY_SIZE(stats.by_size_log); i++) { + uptr c = stats.by_size_log[i]; + if (!c) continue; + Printf("%zd:%zd; ", i, c); + } + Printf("\n"); + } + + // ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone + // introspection API. + void ForceLock() { + mutex_.Lock(); + } + + void ForceUnlock() { + mutex_.Unlock(); + } + + // Iterate over all existing chunks. + // The allocator must be locked when calling this function. + void ForEachChunk(ForEachChunkCallback callback, void *arg) { + for (uptr i = 0; i < n_chunks_; i++) + callback(reinterpret_cast<uptr>(GetUser(chunks_[i])), arg); + } + + private: + static const int kMaxNumChunks = 1 << FIRST_32_SECOND_64(15, 18); + struct Header { + uptr map_beg; + uptr map_size; + uptr size; + uptr chunk_idx; + }; + + Header *GetHeader(uptr p) { + CHECK(IsAligned(p, page_size_)); + return reinterpret_cast<Header*>(p - page_size_); + } + Header *GetHeader(const void *p) { + return GetHeader(reinterpret_cast<uptr>(p)); + } + + void *GetUser(Header *h) { + CHECK(IsAligned((uptr)h, page_size_)); + return reinterpret_cast<void*>(reinterpret_cast<uptr>(h) + page_size_); + } + + uptr RoundUpMapSize(uptr size) { + return RoundUpTo(size, page_size_) + page_size_; + } + + uptr page_size_; + Header *chunks_[kMaxNumChunks]; + uptr n_chunks_; + uptr min_mmap_, max_mmap_; + bool chunks_sorted_; + struct Stats { + uptr n_allocs, n_frees, currently_allocated, max_allocated, by_size_log[64]; + } stats; + atomic_uint8_t may_return_null_; + SpinMutex mutex_; +}; diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_size_class_map.h b/libsanitizer/sanitizer_common/sanitizer_allocator_size_class_map.h new file mode 100644 index 000000000000..4cda021a947a --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_allocator_size_class_map.h @@ -0,0 +1,215 @@ +//===-- sanitizer_allocator_size_class_map.h --------------------*- C++ -*-===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Part of the Sanitizer Allocator. +// +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_ALLOCATOR_H +#error This file must be included inside sanitizer_allocator.h +#endif + +// SizeClassMap maps allocation sizes into size classes and back. +// Class 0 always corresponds to size 0. +// The other sizes are controlled by the template parameters: +// kMinSizeLog: defines the class 1 as 2^kMinSizeLog. +// kMaxSizeLog: defines the last class as 2^kMaxSizeLog. +// kMidSizeLog: the classes starting from 1 increase with step +// 2^kMinSizeLog until 2^kMidSizeLog. +// kNumBits: the number of non-zero bits in sizes after 2^kMidSizeLog. +// E.g. with kNumBits==3 all size classes after 2^kMidSizeLog +// look like 0b1xx0..0, where x is either 0 or 1. +// +// Example: kNumBits=3, kMidSizeLog=4, kMidSizeLog=8, kMaxSizeLog=17: +// +// Classes 1 - 16 correspond to sizes 16 to 256 (size = class_id * 16). +// Next 4 classes: 256 + i * 64 (i = 1 to 4). +// Next 4 classes: 512 + i * 128 (i = 1 to 4). +// ... +// Next 4 classes: 2^k + i * 2^(k-2) (i = 1 to 4). +// Last class corresponds to kMaxSize = 1 << kMaxSizeLog. +// +// This structure of the size class map gives us: +// - Efficient table-free class-to-size and size-to-class functions. +// - Difference between two consequent size classes is between 14% and 25% +// +// This class also gives a hint to a thread-caching allocator about the amount +// of chunks that need to be cached per-thread: +// - kMaxNumCachedHint is a hint for maximal number of chunks per size class. +// The actual number is computed in TransferBatch. +// - (1 << kMaxBytesCachedLog) is the maximal number of bytes per size class. +// +// Part of output of SizeClassMap::Print(): +// c00 => s: 0 diff: +0 00% l 0 cached: 0 0; id 0 +// c01 => s: 16 diff: +16 00% l 4 cached: 256 4096; id 1 +// c02 => s: 32 diff: +16 100% l 5 cached: 256 8192; id 2 +// c03 => s: 48 diff: +16 50% l 5 cached: 256 12288; id 3 +// c04 => s: 64 diff: +16 33% l 6 cached: 256 16384; id 4 +// c05 => s: 80 diff: +16 25% l 6 cached: 256 20480; id 5 +// c06 => s: 96 diff: +16 20% l 6 cached: 256 24576; id 6 +// c07 => s: 112 diff: +16 16% l 6 cached: 256 28672; id 7 +// +// c08 => s: 128 diff: +16 14% l 7 cached: 256 32768; id 8 +// c09 => s: 144 diff: +16 12% l 7 cached: 256 36864; id 9 +// c10 => s: 160 diff: +16 11% l 7 cached: 256 40960; id 10 +// c11 => s: 176 diff: +16 10% l 7 cached: 256 45056; id 11 +// c12 => s: 192 diff: +16 09% l 7 cached: 256 49152; id 12 +// c13 => s: 208 diff: +16 08% l 7 cached: 256 53248; id 13 +// c14 => s: 224 diff: +16 07% l 7 cached: 256 57344; id 14 +// c15 => s: 240 diff: +16 07% l 7 cached: 256 61440; id 15 +// +// c16 => s: 256 diff: +16 06% l 8 cached: 256 65536; id 16 +// c17 => s: 320 diff: +64 25% l 8 cached: 204 65280; id 17 +// c18 => s: 384 diff: +64 20% l 8 cached: 170 65280; id 18 +// c19 => s: 448 diff: +64 16% l 8 cached: 146 65408; id 19 +// +// c20 => s: 512 diff: +64 14% l 9 cached: 128 65536; id 20 +// c21 => s: 640 diff: +128 25% l 9 cached: 102 65280; id 21 +// c22 => s: 768 diff: +128 20% l 9 cached: 85 65280; id 22 +// c23 => s: 896 diff: +128 16% l 9 cached: 73 65408; id 23 +// +// c24 => s: 1024 diff: +128 14% l 10 cached: 64 65536; id 24 +// c25 => s: 1280 diff: +256 25% l 10 cached: 51 65280; id 25 +// c26 => s: 1536 diff: +256 20% l 10 cached: 42 64512; id 26 +// c27 => s: 1792 diff: +256 16% l 10 cached: 36 64512; id 27 +// +// ... +// +// c48 => s: 65536 diff: +8192 14% l 16 cached: 1 65536; id 48 +// c49 => s: 81920 diff: +16384 25% l 16 cached: 1 81920; id 49 +// c50 => s: 98304 diff: +16384 20% l 16 cached: 1 98304; id 50 +// c51 => s: 114688 diff: +16384 16% l 16 cached: 1 114688; id 51 +// +// c52 => s: 131072 diff: +16384 14% l 17 cached: 1 131072; id 52 +// +// +// Another example (kNumBits=2): +// c00 => s: 0 diff: +0 00% l 0 cached: 0 0; id 0 +// c01 => s: 32 diff: +32 00% l 5 cached: 64 2048; id 1 +// c02 => s: 64 diff: +32 100% l 6 cached: 64 4096; id 2 +// c03 => s: 96 diff: +32 50% l 6 cached: 64 6144; id 3 +// c04 => s: 128 diff: +32 33% l 7 cached: 64 8192; id 4 +// c05 => s: 160 diff: +32 25% l 7 cached: 64 10240; id 5 +// c06 => s: 192 diff: +32 20% l 7 cached: 64 12288; id 6 +// c07 => s: 224 diff: +32 16% l 7 cached: 64 14336; id 7 +// c08 => s: 256 diff: +32 14% l 8 cached: 64 16384; id 8 +// c09 => s: 384 diff: +128 50% l 8 cached: 42 16128; id 9 +// c10 => s: 512 diff: +128 33% l 9 cached: 32 16384; id 10 +// c11 => s: 768 diff: +256 50% l 9 cached: 21 16128; id 11 +// c12 => s: 1024 diff: +256 33% l 10 cached: 16 16384; id 12 +// c13 => s: 1536 diff: +512 50% l 10 cached: 10 15360; id 13 +// c14 => s: 2048 diff: +512 33% l 11 cached: 8 16384; id 14 +// c15 => s: 3072 diff: +1024 50% l 11 cached: 5 15360; id 15 +// c16 => s: 4096 diff: +1024 33% l 12 cached: 4 16384; id 16 +// c17 => s: 6144 diff: +2048 50% l 12 cached: 2 12288; id 17 +// c18 => s: 8192 diff: +2048 33% l 13 cached: 2 16384; id 18 +// c19 => s: 12288 diff: +4096 50% l 13 cached: 1 12288; id 19 +// c20 => s: 16384 diff: +4096 33% l 14 cached: 1 16384; id 20 +// c21 => s: 24576 diff: +8192 50% l 14 cached: 1 24576; id 21 +// c22 => s: 32768 diff: +8192 33% l 15 cached: 1 32768; id 22 +// c23 => s: 49152 diff: +16384 50% l 15 cached: 1 49152; id 23 +// c24 => s: 65536 diff: +16384 33% l 16 cached: 1 65536; id 24 +// c25 => s: 98304 diff: +32768 50% l 16 cached: 1 98304; id 25 +// c26 => s: 131072 diff: +32768 33% l 17 cached: 1 131072; id 26 + +template <uptr kNumBits, uptr kMinSizeLog, uptr kMidSizeLog, uptr kMaxSizeLog, + uptr kMaxNumCachedHintT, uptr kMaxBytesCachedLog> +class SizeClassMap { + static const uptr kMinSize = 1 << kMinSizeLog; + static const uptr kMidSize = 1 << kMidSizeLog; + static const uptr kMidClass = kMidSize / kMinSize; + static const uptr S = kNumBits - 1; + static const uptr M = (1 << S) - 1; + + public: + // kMaxNumCachedHintT is a power of two. It serves as a hint + // for the size of TransferBatch, the actual size could be a bit smaller. + static const uptr kMaxNumCachedHint = kMaxNumCachedHintT; + COMPILER_CHECK((kMaxNumCachedHint & (kMaxNumCachedHint - 1)) == 0); + + static const uptr kMaxSize = 1UL << kMaxSizeLog; + static const uptr kNumClasses = + kMidClass + ((kMaxSizeLog - kMidSizeLog) << S) + 1; + static const uptr kLargestClassID = kNumClasses - 2; + COMPILER_CHECK(kNumClasses >= 16 && kNumClasses <= 256); + static const uptr kNumClassesRounded = + kNumClasses <= 32 ? 32 : + kNumClasses <= 64 ? 64 : + kNumClasses <= 128 ? 128 : 256; + + static uptr Size(uptr class_id) { + if (class_id <= kMidClass) + return kMinSize * class_id; + class_id -= kMidClass; + uptr t = kMidSize << (class_id >> S); + return t + (t >> S) * (class_id & M); + } + + static uptr ClassID(uptr size) { + if (size <= kMidSize) + return (size + kMinSize - 1) >> kMinSizeLog; + if (size > kMaxSize) return 0; + uptr l = MostSignificantSetBitIndex(size); + uptr hbits = (size >> (l - S)) & M; + uptr lbits = size & ((1 << (l - S)) - 1); + uptr l1 = l - kMidSizeLog; + return kMidClass + (l1 << S) + hbits + (lbits > 0); + } + + static uptr MaxCachedHint(uptr class_id) { + if (class_id == 0) return 0; + uptr n = (1UL << kMaxBytesCachedLog) / Size(class_id); + return Max<uptr>(1, Min(kMaxNumCachedHint, n)); + } + + static void Print() { + uptr prev_s = 0; + uptr total_cached = 0; + for (uptr i = 0; i < kNumClasses; i++) { + uptr s = Size(i); + if (s >= kMidSize / 2 && (s & (s - 1)) == 0) + Printf("\n"); + uptr d = s - prev_s; + uptr p = prev_s ? (d * 100 / prev_s) : 0; + uptr l = s ? MostSignificantSetBitIndex(s) : 0; + uptr cached = MaxCachedHint(i) * s; + Printf("c%02zd => s: %zd diff: +%zd %02zd%% l %zd " + "cached: %zd %zd; id %zd\n", + i, Size(i), d, p, l, MaxCachedHint(i), cached, ClassID(s)); + total_cached += cached; + prev_s = s; + } + Printf("Total cached: %zd\n", total_cached); + } + + static void Validate() { + for (uptr c = 1; c < kNumClasses; c++) { + // Printf("Validate: c%zd\n", c); + uptr s = Size(c); + CHECK_NE(s, 0U); + CHECK_EQ(ClassID(s), c); + if (c != kNumClasses - 1) + CHECK_EQ(ClassID(s + 1), c + 1); + CHECK_EQ(ClassID(s - 1), c); + if (c) + CHECK_GT(Size(c), Size(c-1)); + } + CHECK_EQ(ClassID(kMaxSize + 1), 0); + + for (uptr s = 1; s <= kMaxSize; s++) { + uptr c = ClassID(s); + // Printf("s%zd => c%zd\n", s, c); + CHECK_LT(c, kNumClasses); + CHECK_GE(Size(c), s); + if (c > 0) + CHECK_LT(Size(c-1), s); + } + } +}; + +typedef SizeClassMap<3, 4, 8, 17, 128, 16> DefaultSizeClassMap; +typedef SizeClassMap<3, 4, 8, 17, 64, 14> CompactSizeClassMap; +typedef SizeClassMap<2, 5, 9, 16, 64, 14> VeryCompactSizeClassMap; diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_stats.h b/libsanitizer/sanitizer_common/sanitizer_allocator_stats.h new file mode 100644 index 000000000000..e336b5d365a5 --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_allocator_stats.h @@ -0,0 +1,103 @@ +//===-- sanitizer_allocator_stats.h -----------------------------*- C++ -*-===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Part of the Sanitizer Allocator. +// +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_ALLOCATOR_H +#error This file must be included inside sanitizer_allocator.h +#endif + +// Memory allocator statistics +enum AllocatorStat { + AllocatorStatAllocated, + AllocatorStatMapped, + AllocatorStatCount +}; + +typedef uptr AllocatorStatCounters[AllocatorStatCount]; + +// Per-thread stats, live in per-thread cache. +class AllocatorStats { + public: + void Init() { + internal_memset(this, 0, sizeof(*this)); + } + void InitLinkerInitialized() {} + + void Add(AllocatorStat i, uptr v) { + v += atomic_load(&stats_[i], memory_order_relaxed); + atomic_store(&stats_[i], v, memory_order_relaxed); + } + + void Sub(AllocatorStat i, uptr v) { + v = atomic_load(&stats_[i], memory_order_relaxed) - v; + atomic_store(&stats_[i], v, memory_order_relaxed); + } + + void Set(AllocatorStat i, uptr v) { + atomic_store(&stats_[i], v, memory_order_relaxed); + } + + uptr Get(AllocatorStat i) const { + return atomic_load(&stats_[i], memory_order_relaxed); + } + + private: + friend class AllocatorGlobalStats; + AllocatorStats *next_; + AllocatorStats *prev_; + atomic_uintptr_t stats_[AllocatorStatCount]; +}; + +// Global stats, used for aggregation and querying. +class AllocatorGlobalStats : public AllocatorStats { + public: + void InitLinkerInitialized() { + next_ = this; + prev_ = this; + } + void Init() { + internal_memset(this, 0, sizeof(*this)); + InitLinkerInitialized(); + } + + void Register(AllocatorStats *s) { + SpinMutexLock l(&mu_); + s->next_ = next_; + s->prev_ = this; + next_->prev_ = s; + next_ = s; + } + + void Unregister(AllocatorStats *s) { + SpinMutexLock l(&mu_); + s->prev_->next_ = s->next_; + s->next_->prev_ = s->prev_; + for (int i = 0; i < AllocatorStatCount; i++) + Add(AllocatorStat(i), s->Get(AllocatorStat(i))); + } + + void Get(AllocatorStatCounters s) const { + internal_memset(s, 0, AllocatorStatCount * sizeof(uptr)); + SpinMutexLock l(&mu_); + const AllocatorStats *stats = this; + for (;;) { + for (int i = 0; i < AllocatorStatCount; i++) + s[i] += stats->Get(AllocatorStat(i)); + stats = stats->next_; + if (stats == this) + break; + } + // All stats must be non-negative. + for (int i = 0; i < AllocatorStatCount; i++) + s[i] = ((sptr)s[i]) >= 0 ? s[i] : 0; + } + + private: + mutable SpinMutex mu_; +}; diff --git a/libsanitizer/sanitizer_common/sanitizer_asm.h b/libsanitizer/sanitizer_common/sanitizer_asm.h index 985f59c23c20..737ccbb716db 100644 --- a/libsanitizer/sanitizer_common/sanitizer_asm.h +++ b/libsanitizer/sanitizer_common/sanitizer_asm.h @@ -40,3 +40,17 @@ # define CFI_DEF_CFA(reg, n) # define CFI_RESTORE(reg) #endif + +#if !defined(__APPLE__) +# define ASM_HIDDEN(symbol) .hidden symbol +# define ASM_TYPE_FUNCTION(symbol) .type symbol, @function +# define ASM_SIZE(symbol) .size symbol, .-symbol +# define ASM_TSAN_SYMBOL(symbol) symbol +# define ASM_TSAN_SYMBOL_INTERCEPTOR(symbol) symbol +#else +# define ASM_HIDDEN(symbol) +# define ASM_TYPE_FUNCTION(symbol) +# define ASM_SIZE(symbol) +# define ASM_TSAN_SYMBOL(symbol) _##symbol +# define ASM_TSAN_SYMBOL_INTERCEPTOR(symbol) _wrap_##symbol +#endif diff --git a/libsanitizer/sanitizer_common/sanitizer_atomic_msvc.h b/libsanitizer/sanitizer_common/sanitizer_atomic_msvc.h index 4ac3b90769f2..4ae87142d467 100644 --- a/libsanitizer/sanitizer_common/sanitizer_atomic_msvc.h +++ b/libsanitizer/sanitizer_common/sanitizer_atomic_msvc.h @@ -31,6 +31,10 @@ extern "C" long _InterlockedExchange( // NOLINT extern "C" long _InterlockedExchangeAdd( // NOLINT long volatile * Addend, long Value); // NOLINT #pragma intrinsic(_InterlockedExchangeAdd) +extern "C" char _InterlockedCompareExchange8( // NOLINT + char volatile *Destination, // NOLINT + char Exchange, char Comparand); // NOLINT +#pragma intrinsic(_InterlockedCompareExchange8) extern "C" short _InterlockedCompareExchange16( // NOLINT short volatile *Destination, // NOLINT short Exchange, short Comparand); // NOLINT @@ -169,8 +173,6 @@ INLINE u32 atomic_exchange(volatile atomic_uint32_t *a, return (u32)_InterlockedExchange((volatile long*)&a->val_dont_use, v); } -#ifndef _WIN64 - INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a, u8 *cmp, u8 xchgv, @@ -178,6 +180,10 @@ INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a, (void)mo; DCHECK(!((uptr)a % sizeof(*a))); u8 cmpv = *cmp; +#ifdef _WIN64 + u8 prev = (u8)_InterlockedCompareExchange8( + (volatile char*)&a->val_dont_use, (char)xchgv, (char)cmpv); +#else u8 prev; __asm { mov al, cmpv @@ -186,14 +192,13 @@ INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a, lock cmpxchg [ecx], dl mov prev, al } +#endif if (prev == cmpv) return true; *cmp = prev; return false; } -#endif - INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a, uptr *cmp, uptr xchg, diff --git a/libsanitizer/sanitizer_common/sanitizer_common.cc b/libsanitizer/sanitizer_common/sanitizer_common.cc index 4529e63eba96..b445f613b85c 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common.cc +++ b/libsanitizer/sanitizer_common/sanitizer_common.cc @@ -10,6 +10,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common.h" +#include "sanitizer_allocator_interface.h" #include "sanitizer_allocator_internal.h" #include "sanitizer_flags.h" #include "sanitizer_libc.h" @@ -22,13 +23,7 @@ namespace __sanitizer { const char *SanitizerToolName = "SanitizerTool"; atomic_uint32_t current_verbosity; - -uptr GetPageSizeCached() { - static uptr PageSize; - if (!PageSize) - PageSize = GetPageSize(); - return PageSize; -} +uptr PageSizeCached; StaticSpinMutex report_file_mu; ReportFile report_file = {&report_file_mu, kStderrFd, "", "", 0}; @@ -103,70 +98,13 @@ uptr stoptheworld_tracer_pid = 0; // writing to the same log file. uptr stoptheworld_tracer_ppid = 0; -static const int kMaxNumOfInternalDieCallbacks = 5; -static DieCallbackType InternalDieCallbacks[kMaxNumOfInternalDieCallbacks]; - -bool AddDieCallback(DieCallbackType callback) { - for (int i = 0; i < kMaxNumOfInternalDieCallbacks; i++) { - if (InternalDieCallbacks[i] == nullptr) { - InternalDieCallbacks[i] = callback; - return true; - } - } - return false; -} - -bool RemoveDieCallback(DieCallbackType callback) { - for (int i = 0; i < kMaxNumOfInternalDieCallbacks; i++) { - if (InternalDieCallbacks[i] == callback) { - internal_memmove(&InternalDieCallbacks[i], &InternalDieCallbacks[i + 1], - sizeof(InternalDieCallbacks[0]) * - (kMaxNumOfInternalDieCallbacks - i - 1)); - InternalDieCallbacks[kMaxNumOfInternalDieCallbacks - 1] = nullptr; - return true; - } - } - return false; -} - -static DieCallbackType UserDieCallback; -void SetUserDieCallback(DieCallbackType callback) { - UserDieCallback = callback; -} - -void NORETURN Die() { - if (UserDieCallback) - UserDieCallback(); - for (int i = kMaxNumOfInternalDieCallbacks - 1; i >= 0; i--) { - if (InternalDieCallbacks[i]) - InternalDieCallbacks[i](); - } - if (common_flags()->abort_on_error) - Abort(); - internal__exit(common_flags()->exitcode); -} - -static CheckFailedCallbackType CheckFailedCallback; -void SetCheckFailedCallback(CheckFailedCallbackType callback) { - CheckFailedCallback = callback; -} - -void NORETURN CheckFailed(const char *file, int line, const char *cond, - u64 v1, u64 v2) { - if (CheckFailedCallback) { - CheckFailedCallback(file, line, cond, v1, v2); - } - Report("Sanitizer CHECK failed: %s:%d %s (%lld, %lld)\n", file, line, cond, - v1, v2); - Die(); -} - void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type, - const char *mmap_type, error_t err) { + const char *mmap_type, error_t err, + bool raw_report) { static int recursion_count; - if (recursion_count) { + if (raw_report || recursion_count) { + // If raw report is requested or we went into recursion, just die. // The Report() and CHECK calls below may call mmap recursively and fail. - // If we went into recursion, just die. RawWrite("ERROR: Failed to mmap\n"); Die(); } @@ -174,7 +112,7 @@ void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type, Report("ERROR: %s failed to " "%s 0x%zx (%zd) bytes of %s (error code: %d)\n", SanitizerToolName, mmap_type, size, size, mem_type, err); -#ifndef SANITIZER_GO +#if !SANITIZER_GO DumpProcessMap(); #endif UNREACHABLE("unable to mmap"); @@ -217,6 +155,7 @@ bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size, } typedef bool UptrComparisonFunction(const uptr &a, const uptr &b); +typedef bool U32ComparisonFunction(const u32 &a, const u32 &b); template<class T> static inline bool CompareLess(const T &a, const T &b) { @@ -227,25 +166,8 @@ void SortArray(uptr *array, uptr size) { InternalSort<uptr*, UptrComparisonFunction>(&array, size, CompareLess); } -// We want to map a chunk of address space aligned to 'alignment'. -// We do it by maping a bit more and then unmaping redundant pieces. -// We probably can do it with fewer syscalls in some OS-dependent way. -void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type) { -// uptr PageSize = GetPageSizeCached(); - CHECK(IsPowerOfTwo(size)); - CHECK(IsPowerOfTwo(alignment)); - uptr map_size = size + alignment; - uptr map_res = (uptr)MmapOrDie(map_size, mem_type); - uptr map_end = map_res + map_size; - uptr res = map_res; - if (res & (alignment - 1)) // Not aligned. - res = (map_res + alignment) & ~(alignment - 1); - uptr end = res + size; - if (res != map_res) - UnmapOrDie((void*)map_res, res - map_res); - if (end != map_end) - UnmapOrDie((void*)end, map_end - end); - return (void*)res; +void SortArray(u32 *array, uptr size) { + InternalSort<u32*, U32ComparisonFunction>(&array, size, CompareLess); } const char *StripPathPrefix(const char *filepath, @@ -283,7 +205,7 @@ void ReportErrorSummary(const char *error_message) { __sanitizer_report_error_summary(buff.data()); } -#ifndef SANITIZER_GO +#if !SANITIZER_GO void ReportErrorSummary(const char *error_type, const AddressInfo &info) { if (!common_flags()->print_summary) return; @@ -295,6 +217,40 @@ void ReportErrorSummary(const char *error_type, const AddressInfo &info) { } #endif +// Removes the ANSI escape sequences from the input string (in-place). +void RemoveANSIEscapeSequencesFromString(char *str) { + if (!str) + return; + + // We are going to remove the escape sequences in place. + char *s = str; + char *z = str; + while (*s != '\0') { + CHECK_GE(s, z); + // Skip over ANSI escape sequences with pointer 's'. + if (*s == '\033' && *(s + 1) == '[') { + s = internal_strchrnul(s, 'm'); + if (*s == '\0') { + break; + } + s++; + continue; + } + // 's' now points at a character we want to keep. Copy over the buffer + // content if the escape sequence has been perviously skipped andadvance + // both pointers. + if (s != z) + *z = *s; + + // If we have not seen an escape sequence, just advance both pointers. + z++; + s++; + } + + // Null terminate the string. + *z = '\0'; +} + void LoadedModule::set(const char *module_name, uptr base_address) { clear(); full_name_ = internal_strdup(module_name); @@ -318,9 +274,8 @@ void LoadedModule::addAddressRange(uptr beg, uptr end, bool executable) { } bool LoadedModule::containsAddress(uptr address) const { - for (Iterator iter = ranges(); iter.hasNext();) { - const AddressRange *r = iter.next(); - if (r->beg <= address && address < r->end) + for (const AddressRange &r : ranges()) { + if (r.beg <= address && address < r.end) return true; } return false; @@ -387,6 +342,10 @@ bool TemplateMatch(const char *templ, const char *str) { static const char kPathSeparator = SANITIZER_WINDOWS ? ';' : ':'; char *FindPathToBinary(const char *name) { + if (FileExists(name)) { + return internal_strdup(name); + } + const char *path = GetEnv("PATH"); if (!path) return nullptr; @@ -451,6 +410,53 @@ uptr ReadBinaryNameCached(/*out*/char *buf, uptr buf_len) { return name_len; } +void PrintCmdline() { + char **argv = GetArgv(); + if (!argv) return; + Printf("\nCommand: "); + for (uptr i = 0; argv[i]; ++i) + Printf("%s ", argv[i]); + Printf("\n\n"); +} + +// Malloc hooks. +static const int kMaxMallocFreeHooks = 5; +struct MallocFreeHook { + void (*malloc_hook)(const void *, uptr); + void (*free_hook)(const void *); +}; + +static MallocFreeHook MFHooks[kMaxMallocFreeHooks]; + +void RunMallocHooks(const void *ptr, uptr size) { + for (int i = 0; i < kMaxMallocFreeHooks; i++) { + auto hook = MFHooks[i].malloc_hook; + if (!hook) return; + hook(ptr, size); + } +} + +void RunFreeHooks(const void *ptr) { + for (int i = 0; i < kMaxMallocFreeHooks; i++) { + auto hook = MFHooks[i].free_hook; + if (!hook) return; + hook(ptr); + } +} + +static int InstallMallocFreeHooks(void (*malloc_hook)(const void *, uptr), + void (*free_hook)(const void *)) { + if (!malloc_hook || !free_hook) return 0; + for (int i = 0; i < kMaxMallocFreeHooks; i++) { + if (MFHooks[i].malloc_hook == nullptr) { + MFHooks[i].malloc_hook = malloc_hook; + MFHooks[i].free_hook = free_hook; + return i + 1; + } + } + return 0; +} + } // namespace __sanitizer using namespace __sanitizer; // NOLINT @@ -460,6 +466,11 @@ void __sanitizer_set_report_path(const char *path) { report_file.SetReportPath(path); } +void __sanitizer_set_report_fd(void *fd) { + report_file.fd = (fd_t)reinterpret_cast<uptr>(fd); + report_file.fd_pid = internal_getpid(); +} + void __sanitizer_report_error_summary(const char *error_summary) { Printf("%s\n", error_summary); } @@ -468,4 +479,18 @@ SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_set_death_callback(void (*callback)(void)) { SetUserDieCallback(callback); } + +SANITIZER_INTERFACE_ATTRIBUTE +int __sanitizer_install_malloc_and_free_hooks(void (*malloc_hook)(const void *, + uptr), + void (*free_hook)(const void *)) { + return InstallMallocFreeHooks(malloc_hook, free_hook); +} + +#if !SANITIZER_GO && !SANITIZER_SUPPORTS_WEAK_HOOKS +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +void __sanitizer_print_memory_profile(int top_percent) { + (void)top_percent; +} +#endif } // extern "C" diff --git a/libsanitizer/sanitizer_common/sanitizer_common.h b/libsanitizer/sanitizer_common/sanitizer_common.h index 6fb2dd882aa7..3e079ab7ad1b 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common.h +++ b/libsanitizer/sanitizer_common/sanitizer_common.h @@ -21,7 +21,7 @@ #include "sanitizer_list.h" #include "sanitizer_mutex.h" -#ifdef _MSC_VER +#if defined(_MSC_VER) && !defined(__clang__) extern "C" void _ReadWriteBarrier(); #pragma intrinsic(_ReadWriteBarrier) #endif @@ -42,11 +42,10 @@ const uptr kWordSizeInBits = 8 * kWordSize; const uptr kMaxPathLength = 4096; -// 16K loaded modules should be enough for everyone. -static const uptr kMaxNumberOfModules = 1 << 14; - const uptr kMaxThreadStackSize = 1 << 30; // 1Gb +static const uptr kErrorMessageBufferSize = 1 << 16; + // Denotes fake PC values that come from JIT/JAVA/etc. // For such PC values __tsan_symbolize_external() will be called. const u64 kExternalPCBit = 1ULL << 60; @@ -62,7 +61,12 @@ INLINE int Verbosity() { } uptr GetPageSize(); -uptr GetPageSizeCached(); +extern uptr PageSizeCached; +INLINE uptr GetPageSizeCached() { + if (!PageSizeCached) + PageSizeCached = GetPageSize(); + return PageSizeCached; +} uptr GetMmapGranularity(); uptr GetMaxVirtualAddress(); // Threads @@ -74,22 +78,30 @@ void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, uptr *tls_addr, uptr *tls_size); // Memory management -void *MmapOrDie(uptr size, const char *mem_type); +void *MmapOrDie(uptr size, const char *mem_type, bool raw_report = false); +INLINE void *MmapOrDieQuietly(uptr size, const char *mem_type) { + return MmapOrDie(size, mem_type, /*raw_report*/ true); +} void UnmapOrDie(void *addr, uptr size); void *MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name = nullptr); void *MmapNoReserveOrDie(uptr size, const char *mem_type); void *MmapFixedOrDie(uptr fixed_addr, uptr size); -void *MmapNoAccess(uptr fixed_addr, uptr size, const char *name = nullptr); +void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name = nullptr); +void *MmapNoAccess(uptr size); // Map aligned chunk of address space; size and alignment are powers of two. void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type); -// Disallow access to a memory range. Use MmapNoAccess to allocate an +// Disallow access to a memory range. Use MmapFixedNoAccess to allocate an // unaccessible memory. bool MprotectNoAccess(uptr addr, uptr size); +bool MprotectReadOnly(uptr addr, uptr size); + +// Find an available address space. +uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding); // Used to check if we can map shadow memory to a fixed location. bool MemoryRangeIsAvailable(uptr range_start, uptr range_end); -void FlushUnneededShadowMemory(uptr addr, uptr size); +void ReleaseMemoryToOS(uptr addr, uptr size); void IncreaseTotalMmap(uptr size); void DecreaseTotalMmap(uptr size); uptr GetRSS(); @@ -97,21 +109,21 @@ void NoHugePagesInRegion(uptr addr, uptr length); void DontDumpShadowMemory(uptr addr, uptr length); // Check if the built VMA size matches the runtime one. void CheckVMASize(); +void RunMallocHooks(const void *ptr, uptr size); +void RunFreeHooks(const void *ptr); // InternalScopedBuffer can be used instead of large stack arrays to // keep frame size low. // FIXME: use InternalAlloc instead of MmapOrDie once // InternalAlloc is made libc-free. -template<typename T> +template <typename T> class InternalScopedBuffer { public: explicit InternalScopedBuffer(uptr cnt) { cnt_ = cnt; - ptr_ = (T*)MmapOrDie(cnt * sizeof(T), "InternalScopedBuffer"); - } - ~InternalScopedBuffer() { - UnmapOrDie(ptr_, cnt_ * sizeof(T)); + ptr_ = (T *)MmapOrDie(cnt * sizeof(T), "InternalScopedBuffer"); } + ~InternalScopedBuffer() { UnmapOrDie(ptr_, cnt_ * sizeof(T)); } T &operator[](uptr i) { return ptr_[i]; } T *data() { return ptr_; } uptr size() { return cnt_ * sizeof(T); } @@ -119,9 +131,11 @@ class InternalScopedBuffer { private: T *ptr_; uptr cnt_; - // Disallow evil constructors. - InternalScopedBuffer(const InternalScopedBuffer&); - void operator=(const InternalScopedBuffer&); + // Disallow copies and moves. + InternalScopedBuffer(const InternalScopedBuffer &) = delete; + InternalScopedBuffer &operator=(const InternalScopedBuffer &) = delete; + InternalScopedBuffer(InternalScopedBuffer &&) = delete; + InternalScopedBuffer &operator=(InternalScopedBuffer &&) = delete; }; class InternalScopedString : public InternalScopedBuffer<char> { @@ -160,6 +174,7 @@ void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback); // IO void RawWrite(const char *buffer); bool ColorizeReports(); +void RemoveANSIEscapeSequencesFromString(char *buffer); void Printf(const char *format, ...); void Report(const char *format, ...); void SetPrintfAndReportCallback(void (*callback)(const char *)); @@ -271,10 +286,27 @@ const char *GetPwd(); char *FindPathToBinary(const char *name); bool IsPathSeparator(const char c); bool IsAbsolutePath(const char *path); +// Starts a subprocess and returs its pid. +// If *_fd parameters are not kInvalidFd their corresponding input/output +// streams will be redirect to the file. The files will always be closed +// in parent process even in case of an error. +// The child process will close all fds after STDERR_FILENO +// before passing control to a program. +pid_t StartSubprocess(const char *filename, const char *const argv[], + fd_t stdin_fd = kInvalidFd, fd_t stdout_fd = kInvalidFd, + fd_t stderr_fd = kInvalidFd); +// Checks if specified process is still running +bool IsProcessRunning(pid_t pid); +// Waits for the process to finish and returns its exit code. +// Returns -1 in case of an error. +int WaitForProcess(pid_t pid); u32 GetUid(); void ReExec(); +char **GetArgv(); +void PrintCmdline(); bool StackSizeIsUnlimited(); +uptr GetStackSizeLimitInBytes(); void SetStackSizeLimitInBytes(uptr limit); bool AddressSpaceIsUnlimited(); void SetAddressSpaceUnlimited(); @@ -299,6 +331,7 @@ void SleepForMillis(int millis); u64 NanoTime(); int Atexit(void (*function)(void)); void SortArray(uptr *array, uptr size); +void SortArray(u32 *array, uptr size); bool TemplateMatch(const char *templ, const char *str); // Exit @@ -307,7 +340,8 @@ void NORETURN Die(); void NORETURN CheckFailed(const char *file, int line, const char *cond, u64 v1, u64 v2); void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type, - const char *mmap_type, error_t err); + const char *mmap_type, error_t err, + bool raw_report = false); // Set the name of the current thread to 'name', return true on succees. // The name may be truncated to a system-dependent limit. @@ -339,9 +373,15 @@ void SetCheckFailedCallback(CheckFailedCallbackType callback); // The callback should be registered once at the tool init time. void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded)); +// Callback to be called when we want to try releasing unused allocator memory +// back to the OS. +typedef void (*AllocatorReleaseToOSCallback)(); +// The callback should be registered once at the tool init time. +void SetAllocatorReleaseToOSCallback(AllocatorReleaseToOSCallback Callback); + // Functions related to signal handling. typedef void (*SignalHandlerType)(int, void *, void *); -bool IsDeadlySignal(int signum); +bool IsHandledDeadlySignal(int signum); void InstallDeadlySignalHandlers(SignalHandlerType handler); // Alternative signal stack (POSIX-only). void SetAlternateSignalStack(); @@ -357,7 +397,7 @@ void ReportErrorSummary(const char *error_message); // error_type file:line[:column][ function] void ReportErrorSummary(const char *error_type, const AddressInfo &info); // Same as above, but obtains AddressInfo by symbolizing top stack trace frame. -void ReportErrorSummary(const char *error_type, StackTrace *trace); +void ReportErrorSummary(const char *error_type, const StackTrace *trace); // Math #if SANITIZER_WINDOWS && !defined(__clang__) && !defined(__GNUC__) @@ -414,13 +454,13 @@ INLINE uptr RoundUpToPowerOfTwo(uptr size) { if (IsPowerOfTwo(size)) return size; uptr up = MostSignificantSetBitIndex(size); - CHECK(size < (1ULL << (up + 1))); - CHECK(size > (1ULL << up)); + CHECK_LT(size, (1ULL << (up + 1))); + CHECK_GT(size, (1ULL << up)); return 1ULL << (up + 1); } INLINE uptr RoundUpTo(uptr size, uptr boundary) { - CHECK(IsPowerOfTwo(boundary)); + RAW_CHECK(IsPowerOfTwo(boundary)); return (size + boundary - 1) & ~(boundary - 1); } @@ -487,7 +527,7 @@ class InternalMmapVectorNoCtor { uptr new_capacity = RoundUpToPowerOfTwo(size_ + 1); Resize(new_capacity); } - data_[size_++] = element; + internal_memcpy(&data_[size_++], &element, sizeof(T)); } T &back() { CHECK_GT(size_, 0); @@ -513,6 +553,19 @@ class InternalMmapVectorNoCtor { void clear() { size_ = 0; } bool empty() const { return size() == 0; } + const T *begin() const { + return data(); + } + T *begin() { + return data(); + } + const T *end() const { + return data() + size(); + } + T *end() { + return data() + size(); + } + private: void Resize(uptr new_capacity) { CHECK_GT(new_capacity, 0); @@ -619,8 +672,7 @@ class LoadedModule { : next(nullptr), beg(beg), end(end), executable(executable) {} }; - typedef IntrusiveList<AddressRange>::ConstIterator Iterator; - Iterator ranges() const { return Iterator(&ranges_); } + const IntrusiveList<AddressRange> &ranges() const { return ranges_; } private: char *full_name_; // Owned. @@ -628,13 +680,33 @@ class LoadedModule { IntrusiveList<AddressRange> ranges_; }; -// OS-dependent function that fills array with descriptions of at most -// "max_modules" currently loaded modules. Returns the number of -// initialized modules. If filter is nonzero, ignores modules for which -// filter(full_name) is false. -typedef bool (*string_predicate_t)(const char *); -uptr GetListOfModules(LoadedModule *modules, uptr max_modules, - string_predicate_t filter); +// List of LoadedModules. OS-dependent implementation is responsible for +// filling this information. +class ListOfModules { + public: + ListOfModules() : modules_(kInitialCapacity) {} + ~ListOfModules() { clear(); } + void init(); + const LoadedModule *begin() const { return modules_.begin(); } + LoadedModule *begin() { return modules_.begin(); } + const LoadedModule *end() const { return modules_.end(); } + LoadedModule *end() { return modules_.end(); } + uptr size() const { return modules_.size(); } + const LoadedModule &operator[](uptr i) const { + CHECK_LT(i, modules_.size()); + return modules_[i]; + } + + private: + void clear() { + for (auto &module : modules_) module.clear(); + modules_.clear(); + } + + InternalMmapVector<LoadedModule> modules_; + // We rarely have more than 16K loaded modules. + static const uptr kInitialCapacity = 1 << 14; +}; // Callback type for iterating over a set of memory ranges. typedef void (*RangeIteratorCallback)(uptr begin, uptr end, void *arg); @@ -646,22 +718,34 @@ enum AndroidApiLevel { ANDROID_POST_LOLLIPOP = 23 }; +void WriteToSyslog(const char *buffer); + +#if SANITIZER_MAC +void LogFullErrorReport(const char *buffer); +#else +INLINE void LogFullErrorReport(const char *buffer) {} +#endif + +#if SANITIZER_LINUX || SANITIZER_MAC +void WriteOneLineToSyslog(const char *s); +void LogMessageOnPrintf(const char *str); +#else +INLINE void WriteOneLineToSyslog(const char *s) {} +INLINE void LogMessageOnPrintf(const char *str) {} +#endif + #if SANITIZER_LINUX // Initialize Android logging. Any writes before this are silently lost. void AndroidLogInit(); -void WriteToSyslog(const char *buffer); #else INLINE void AndroidLogInit() {} -INLINE void WriteToSyslog(const char *buffer) {} #endif #if SANITIZER_ANDROID -void GetExtraActivationFlags(char *buf, uptr size); void SanitizerInitializeUnwinder(); AndroidApiLevel AndroidGetApiLevel(); #else INLINE void AndroidLogWrite(const char *buffer_unused) {} -INLINE void GetExtraActivationFlags(char *buf, uptr size) { *buf = '\0'; } INLINE void SanitizerInitializeUnwinder() {} INLINE AndroidApiLevel AndroidGetApiLevel() { return ANDROID_NOT_ANDROID; } #endif @@ -686,7 +770,7 @@ void MaybeStartBackgroudThread(); // compiler from recognising it and turning it into an actual call to // memset/memcpy/etc. static inline void SanitizerBreakOptimization(void *arg) { -#if _MSC_VER && !defined(__clang__) +#if defined(_MSC_VER) && !defined(__clang__) _ReadWriteBarrier(); #else __asm__ __volatile__("" : : "r" (arg) : "memory"); @@ -699,27 +783,68 @@ struct SignalContext { uptr pc; uptr sp; uptr bp; + bool is_memory_access; - SignalContext(void *context, uptr addr, uptr pc, uptr sp, uptr bp) : - context(context), addr(addr), pc(pc), sp(sp), bp(bp) { - } + enum WriteFlag { UNKNOWN, READ, WRITE } write_flag; + + SignalContext(void *context, uptr addr, uptr pc, uptr sp, uptr bp, + bool is_memory_access, WriteFlag write_flag) + : context(context), + addr(addr), + pc(pc), + sp(sp), + bp(bp), + is_memory_access(is_memory_access), + write_flag(write_flag) {} // Creates signal context in a platform-specific manner. static SignalContext Create(void *siginfo, void *context); + + // Returns true if the "context" indicates a memory write. + static WriteFlag GetWriteFlag(void *context); }; void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp); -} // namespace __sanitizer +void MaybeReexec(); -inline void *operator new(__sanitizer::operator_new_size_type size, - __sanitizer::LowLevelAllocator &alloc) { - return alloc.Allocate(size); +template <typename Fn> +class RunOnDestruction { + public: + explicit RunOnDestruction(Fn fn) : fn_(fn) {} + ~RunOnDestruction() { fn_(); } + + private: + Fn fn_; +}; + +// A simple scope guard. Usage: +// auto cleanup = at_scope_exit([]{ do_cleanup; }); +template <typename Fn> +RunOnDestruction<Fn> at_scope_exit(Fn fn) { + return RunOnDestruction<Fn>(fn); } +// Linux on 64-bit s390 had a nasty bug that crashes the whole machine +// if a process uses virtual memory over 4TB (as many sanitizers like +// to do). This function will abort the process if running on a kernel +// that looks vulnerable. +#if SANITIZER_LINUX && SANITIZER_S390_64 +void AvoidCVE_2016_2143(); +#else +INLINE void AvoidCVE_2016_2143() {} +#endif + struct StackDepotStats { uptr n_uniq_ids; uptr allocated; }; +} // namespace __sanitizer + +inline void *operator new(__sanitizer::operator_new_size_type size, + __sanitizer::LowLevelAllocator &alloc) { + return alloc.Allocate(size); +} + #endif // SANITIZER_COMMON_H diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc b/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc index 8223778cac42..38390ed96215 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc +++ b/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc @@ -9,7 +9,7 @@ // ThreadSanitizer, MemorySanitizer, etc. // // This file should be included into the tool's interceptor file, -// which has to define it's own macros: +// which has to define its own macros: // COMMON_INTERCEPTOR_ENTER // COMMON_INTERCEPTOR_ENTER_NOIGNORE // COMMON_INTERCEPTOR_READ_RANGE @@ -89,6 +89,10 @@ #define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) {} #endif +#ifndef COMMON_INTERCEPTOR_MUTEX_INVALID +#define COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m) {} +#endif + #ifndef COMMON_INTERCEPTOR_HANDLE_RECVMSG #define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) ((void)(msg)) #endif @@ -141,6 +145,22 @@ #define COMMON_INTERCEPTOR_RELEASE(ctx, u) {} #endif +#ifndef COMMON_INTERCEPTOR_USER_CALLBACK_START +#define COMMON_INTERCEPTOR_USER_CALLBACK_START() {} +#endif + +#ifndef COMMON_INTERCEPTOR_USER_CALLBACK_END +#define COMMON_INTERCEPTOR_USER_CALLBACK_END() {} +#endif + +#ifdef SANITIZER_NLDBL_VERSION +#define COMMON_INTERCEPT_FUNCTION_LDBL(fn) \ + COMMON_INTERCEPT_FUNCTION_VER(fn, SANITIZER_NLDBL_VERSION) +#else +#define COMMON_INTERCEPT_FUNCTION_LDBL(fn) \ + COMMON_INTERCEPT_FUNCTION(fn) +#endif + struct FileMetadata { // For open_memstream(). char **addr; @@ -190,6 +210,40 @@ UNUSED static void DeleteInterceptorMetadata(void *addr) { } #endif // SI_NOT_WINDOWS +#if SANITIZER_INTERCEPT_STRLEN +INTERCEPTOR(SIZE_T, strlen, const char *s) { + // Sometimes strlen is called prior to InitializeCommonInterceptors, + // in which case the REAL(strlen) typically used in + // COMMON_INTERCEPTOR_ENTER will fail. We use internal_strlen here + // to handle that. + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) + return internal_strlen(s); + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strlen, s); + SIZE_T result = REAL(strlen)(s); + if (common_flags()->intercept_strlen) + COMMON_INTERCEPTOR_READ_RANGE(ctx, s, result + 1); + return result; +} +#define INIT_STRLEN COMMON_INTERCEPT_FUNCTION(strlen) +#else +#define INIT_STRLEN +#endif + +#if SANITIZER_INTERCEPT_STRNLEN +INTERCEPTOR(SIZE_T, strnlen, const char *s, SIZE_T maxlen) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strnlen, s, maxlen); + SIZE_T length = REAL(strnlen)(s, maxlen); + if (common_flags()->intercept_strlen) + COMMON_INTERCEPTOR_READ_RANGE(ctx, s, Min(length + 1, maxlen)); + return length; +} +#define INIT_STRNLEN COMMON_INTERCEPT_FUNCTION(strnlen) +#else +#define INIT_STRNLEN +#endif + #if SANITIZER_INTERCEPT_TEXTDOMAIN INTERCEPTOR(char*, textdomain, const char *domainname) { void *ctx; @@ -212,13 +266,11 @@ static inline int CharCmpX(unsigned char c1, unsigned char c2) { } DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strcmp, uptr called_pc, - const char *s1, const char *s2) + const char *s1, const char *s2, int result) INTERCEPTOR(int, strcmp, const char *s1, const char *s2) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, strcmp, s1, s2); - CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strcmp, GET_CALLER_PC(), s1, - s2); unsigned char c1, c2; uptr i; for (i = 0;; i++) { @@ -228,19 +280,21 @@ INTERCEPTOR(int, strcmp, const char *s1, const char *s2) { } COMMON_INTERCEPTOR_READ_STRING(ctx, s1, i + 1); COMMON_INTERCEPTOR_READ_STRING(ctx, s2, i + 1); - return CharCmpX(c1, c2); + int result = CharCmpX(c1, c2); + CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strcmp, GET_CALLER_PC(), s1, + s2, result); + return result; } DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strncmp, uptr called_pc, - const char *s1, const char *s2, uptr n) + const char *s1, const char *s2, uptr n, + int result) INTERCEPTOR(int, strncmp, const char *s1, const char *s2, uptr size) { if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) return internal_strncmp(s1, s2, size); void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, strncmp, s1, s2, size); - CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strncmp, GET_CALLER_PC(), s1, - s2, size); unsigned char c1 = 0, c2 = 0; uptr i; for (i = 0; i < size; i++) { @@ -248,9 +302,12 @@ INTERCEPTOR(int, strncmp, const char *s1, const char *s2, uptr size) { c2 = (unsigned char)s2[i]; if (c1 != c2 || c1 == '\0') break; } - COMMON_INTERCEPTOR_READ_RANGE(ctx, s1, Min(i + 1, size)); - COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, Min(i + 1, size)); - return CharCmpX(c1, c2); + COMMON_INTERCEPTOR_READ_STRING(ctx, s1, Min(i + 1, size)); + COMMON_INTERCEPTOR_READ_STRING(ctx, s2, Min(i + 1, size)); + int result = CharCmpX(c1, c2); + CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strncmp, GET_CALLER_PC(), s1, + s2, size, result); + return result; } #define INIT_STRCMP COMMON_INTERCEPT_FUNCTION(strcmp) @@ -267,6 +324,9 @@ static inline int CharCaseCmp(unsigned char c1, unsigned char c2) { return c1_low - c2_low; } +DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strcasecmp, uptr called_pc, + const char *s1, const char *s2, int result) + INTERCEPTOR(int, strcasecmp, const char *s1, const char *s2) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, strcasecmp, s1, s2); @@ -279,9 +339,16 @@ INTERCEPTOR(int, strcasecmp, const char *s1, const char *s2) { } COMMON_INTERCEPTOR_READ_STRING(ctx, s1, i + 1); COMMON_INTERCEPTOR_READ_STRING(ctx, s2, i + 1); - return CharCaseCmp(c1, c2); + int result = CharCaseCmp(c1, c2); + CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strcasecmp, GET_CALLER_PC(), + s1, s2, result); + return result; } +DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strncasecmp, uptr called_pc, + const char *s1, const char *s2, uptr n, + int result) + INTERCEPTOR(int, strncasecmp, const char *s1, const char *s2, SIZE_T n) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, strncasecmp, s1, s2, n); @@ -294,7 +361,10 @@ INTERCEPTOR(int, strncasecmp, const char *s1, const char *s2, SIZE_T n) { } COMMON_INTERCEPTOR_READ_RANGE(ctx, s1, Min(i + 1, n)); COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, Min(i + 1, n)); - return CharCaseCmp(c1, c2); + int result = CharCaseCmp(c1, c2); + CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strncasecmp, GET_CALLER_PC(), + s1, s2, n, result); + return result; } #define INIT_STRCASECMP COMMON_INTERCEPT_FUNCTION(strcasecmp) @@ -316,6 +386,10 @@ static inline void StrstrCheck(void *ctx, char *r, const char *s1, #endif #if SANITIZER_INTERCEPT_STRSTR + +DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strstr, uptr called_pc, + const char *s1, const char *s2, char *result) + INTERCEPTOR(char*, strstr, const char *s1, const char *s2) { if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) return internal_strstr(s1, s2); @@ -324,6 +398,8 @@ INTERCEPTOR(char*, strstr, const char *s1, const char *s2) { char *r = REAL(strstr)(s1, s2); if (common_flags()->intercept_strstr) StrstrCheck(ctx, r, s1, s2); + CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strstr, GET_CALLER_PC(), s1, + s2, r); return r; } @@ -333,12 +409,18 @@ INTERCEPTOR(char*, strstr, const char *s1, const char *s2) { #endif #if SANITIZER_INTERCEPT_STRCASESTR + +DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strcasestr, uptr called_pc, + const char *s1, const char *s2, char *result) + INTERCEPTOR(char*, strcasestr, const char *s1, const char *s2) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, strcasestr, s1, s2); char *r = REAL(strcasestr)(s1, s2); if (common_flags()->intercept_strstr) StrstrCheck(ctx, r, s1, s2); + CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strcasestr, GET_CALLER_PC(), + s1, s2, r); return r; } @@ -347,6 +429,79 @@ INTERCEPTOR(char*, strcasestr, const char *s1, const char *s2) { #define INIT_STRCASESTR #endif +#if SANITIZER_INTERCEPT_MEMMEM +DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memmem, uptr called_pc, + const void *s1, SIZE_T len1, const void *s2, + SIZE_T len2, void *result) + +INTERCEPTOR(void*, memmem, const void *s1, SIZE_T len1, const void *s2, + SIZE_T len2) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, memmem, s1, len1, s2, len2); + void *r = REAL(memmem)(s1, len1, s2, len2); + if (common_flags()->intercept_memmem) { + COMMON_INTERCEPTOR_READ_RANGE(ctx, s1, len1); + COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, len2); + } + CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memmem, GET_CALLER_PC(), + s1, len1, s2, len2, r); + return r; +} + +#define INIT_MEMMEM COMMON_INTERCEPT_FUNCTION(memmem); +#else +#define INIT_MEMMEM +#endif // SANITIZER_INTERCEPT_MEMMEM + +#if SANITIZER_INTERCEPT_STRCHR +INTERCEPTOR(char*, strchr, const char *s, int c) { + void *ctx; + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) + return internal_strchr(s, c); + COMMON_INTERCEPTOR_ENTER(ctx, strchr, s, c); + char *result = REAL(strchr)(s, c); + uptr len = internal_strlen(s); + uptr n = result ? result - s + 1 : len + 1; + if (common_flags()->intercept_strchr) + COMMON_INTERCEPTOR_READ_STRING_OF_LEN(ctx, s, len, n); + return result; +} +#define INIT_STRCHR COMMON_INTERCEPT_FUNCTION(strchr) +#else +#define INIT_STRCHR +#endif + +#if SANITIZER_INTERCEPT_STRCHRNUL +INTERCEPTOR(char*, strchrnul, const char *s, int c) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strchrnul, s, c); + char *result = REAL(strchrnul)(s, c); + uptr len = result - s + 1; + if (common_flags()->intercept_strchr) + COMMON_INTERCEPTOR_READ_STRING(ctx, s, len); + return result; +} +#define INIT_STRCHRNUL COMMON_INTERCEPT_FUNCTION(strchrnul) +#else +#define INIT_STRCHRNUL +#endif + +#if SANITIZER_INTERCEPT_STRRCHR +INTERCEPTOR(char*, strrchr, const char *s, int c) { + void *ctx; + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) + return internal_strrchr(s, c); + COMMON_INTERCEPTOR_ENTER(ctx, strrchr, s, c); + uptr len = internal_strlen(s); + if (common_flags()->intercept_strchr) + COMMON_INTERCEPTOR_READ_STRING_OF_LEN(ctx, s, len, len + 1); + return REAL(strrchr)(s, c); +} +#define INIT_STRRCHR COMMON_INTERCEPT_FUNCTION(strrchr) +#else +#define INIT_STRRCHR +#endif + #if SANITIZER_INTERCEPT_STRSPN INTERCEPTOR(SIZE_T, strspn, const char *s1, const char *s2) { void *ctx; @@ -395,18 +550,75 @@ INTERCEPTOR(char *, strpbrk, const char *s1, const char *s2) { #define INIT_STRPBRK #endif +#if SANITIZER_INTERCEPT_MEMSET +INTERCEPTOR(void*, memset, void *dst, int v, uptr size) { + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) + return internal_memset(dst, v, size); + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size); + if (common_flags()->intercept_intrin) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); + return REAL(memset)(dst, v, size); +} + +#define INIT_MEMSET COMMON_INTERCEPT_FUNCTION(memset) +#else +#define INIT_MEMSET +#endif + +#if SANITIZER_INTERCEPT_MEMMOVE +INTERCEPTOR(void*, memmove, void *dst, const void *src, uptr size) { + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) + return internal_memmove(dst, src, size); + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, memmove, dst, src, size); + if (common_flags()->intercept_intrin) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size); + } + return REAL(memmove)(dst, src, size); +} + +#define INIT_MEMMOVE COMMON_INTERCEPT_FUNCTION(memmove) +#else +#define INIT_MEMMOVE +#endif + +#if SANITIZER_INTERCEPT_MEMCPY +INTERCEPTOR(void*, memcpy, void *dst, const void *src, uptr size) { + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) { + // On OS X, calling internal_memcpy here will cause memory corruptions, + // because memcpy and memmove are actually aliases of the same + // implementation. We need to use internal_memmove here. + return internal_memmove(dst, src, size); + } + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, memcpy, dst, src, size); + if (common_flags()->intercept_intrin) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size); + } + // N.B.: If we switch this to internal_ we'll have to use internal_memmove + // due to memcpy being an alias of memmove on OS X. + return REAL(memcpy)(dst, src, size); +} + +#define INIT_MEMCPY COMMON_INTERCEPT_FUNCTION(memcpy) +#else +#define INIT_MEMCPY +#endif + #if SANITIZER_INTERCEPT_MEMCMP DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, uptr called_pc, - const void *s1, const void *s2, uptr n) + const void *s1, const void *s2, uptr n, + int result) INTERCEPTOR(int, memcmp, const void *a1, const void *a2, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, memcmp, a1, a2, size); if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) return internal_memcmp(a1, a2, size); - CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, GET_CALLER_PC(), a1, - a2, size); + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, memcmp, a1, a2, size); if (common_flags()->intercept_memcmp) { if (common_flags()->strict_memcmp) { // Check the entire regions even if the first bytes of the buffers are @@ -426,10 +638,16 @@ INTERCEPTOR(int, memcmp, const void *a1, const void *a2, uptr size) { } COMMON_INTERCEPTOR_READ_RANGE(ctx, s1, Min(i + 1, size)); COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, Min(i + 1, size)); - return CharCmpX(c1, c2); + int r = CharCmpX(c1, c2); + CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, GET_CALLER_PC(), + a1, a2, size, r); + return r; } } - return REAL(memcmp(a1, a2, size)); + int result = REAL(memcmp(a1, a2, size)); + CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, GET_CALLER_PC(), a1, + a2, size, result); + return result; } #define INIT_MEMCMP COMMON_INTERCEPT_FUNCTION(memcmp) @@ -443,7 +661,16 @@ INTERCEPTOR(void*, memchr, const void *s, int c, SIZE_T n) { return internal_memchr(s, c, n); void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, memchr, s, c, n); +#if SANITIZER_WINDOWS + void *res; + if (REAL(memchr)) { + res = REAL(memchr)(s, c, n); + } else { + res = internal_memchr(s, c, n); + } +#else void *res = REAL(memchr)(s, c, n); +#endif uptr len = res ? (char *)res - (const char *)s + 1 : n; COMMON_INTERCEPTOR_READ_RANGE(ctx, s, len); return res; @@ -488,7 +715,7 @@ INTERCEPTOR(float, frexpf, float x, int *exp) { COMMON_INTERCEPTOR_ENTER(ctx, frexpf, x, exp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. float res = REAL(frexpf)(x, exp); COMMON_INTERCEPTOR_WRITE_RANGE(ctx, exp, sizeof(*exp)); return res; @@ -499,7 +726,7 @@ INTERCEPTOR(long double, frexpl, long double x, int *exp) { COMMON_INTERCEPTOR_ENTER(ctx, frexpl, x, exp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. long double res = REAL(frexpl)(x, exp); COMMON_INTERCEPTOR_WRITE_RANGE(ctx, exp, sizeof(*exp)); return res; @@ -507,7 +734,7 @@ INTERCEPTOR(long double, frexpl, long double x, int *exp) { #define INIT_FREXPF_FREXPL \ COMMON_INTERCEPT_FUNCTION(frexpf); \ - COMMON_INTERCEPT_FUNCTION(frexpl) + COMMON_INTERCEPT_FUNCTION_LDBL(frexpl) #else #define INIT_FREXPF_FREXPL #endif // SANITIZER_INTERCEPT_FREXPF_FREXPL @@ -540,7 +767,7 @@ INTERCEPTOR(SSIZE_T, read, int fd, void *ptr, SIZE_T count) { COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SSIZE_T res = REAL(read)(fd, ptr, count); if (res > 0) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res); if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd); @@ -558,7 +785,7 @@ INTERCEPTOR(SSIZE_T, pread, int fd, void *ptr, SIZE_T count, OFF_T offset) { COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SSIZE_T res = REAL(pread)(fd, ptr, count, offset); if (res > 0) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res); if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd); @@ -576,7 +803,7 @@ INTERCEPTOR(SSIZE_T, pread64, int fd, void *ptr, SIZE_T count, OFF64_T offset) { COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SSIZE_T res = REAL(pread64)(fd, ptr, count, offset); if (res > 0) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res); if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd); @@ -823,7 +1050,7 @@ INTERCEPTOR(char *, ctime, unsigned long *timep) { COMMON_INTERCEPTOR_ENTER(ctx, ctime, timep); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(ctime)(timep); if (res) { COMMON_INTERCEPTOR_READ_RANGE(ctx, timep, sizeof(*timep)); @@ -836,7 +1063,7 @@ INTERCEPTOR(char *, ctime_r, unsigned long *timep, char *result) { COMMON_INTERCEPTOR_ENTER(ctx, ctime_r, timep, result); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(ctime_r)(timep, result); if (res) { COMMON_INTERCEPTOR_READ_RANGE(ctx, timep, sizeof(*timep)); @@ -849,7 +1076,7 @@ INTERCEPTOR(char *, asctime, __sanitizer_tm *tm) { COMMON_INTERCEPTOR_ENTER(ctx, asctime, tm); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(asctime)(tm); if (res) { COMMON_INTERCEPTOR_READ_RANGE(ctx, tm, sizeof(*tm)); @@ -862,7 +1089,7 @@ INTERCEPTOR(char *, asctime_r, __sanitizer_tm *tm, char *result) { COMMON_INTERCEPTOR_ENTER(ctx, asctime_r, tm, result); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(asctime_r)(tm, result); if (res) { COMMON_INTERCEPTOR_READ_RANGE(ctx, tm, sizeof(*tm)); @@ -906,7 +1133,7 @@ INTERCEPTOR(char *, strptime, char *s, char *format, __sanitizer_tm *tm) { COMMON_INTERCEPTOR_READ_RANGE(ctx, format, REAL(strlen)(format) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(strptime)(s, format, tm); COMMON_INTERCEPTOR_READ_STRING(ctx, s, res ? res - s : 0); if (res && tm) { @@ -1043,7 +1270,7 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_sscanf, __isoc99_vsscanf, str, format) // FIXME: under ASan the REAL() call below may write to freed memory and // corrupt its metadata. See -// https://code.google.com/p/address-sanitizer/issues/detail?id=321. +// https://github.com/google/sanitizers/issues/321. #define VSPRINTF_INTERCEPTOR_IMPL(vname, str, ...) \ { \ VPRINTF_INTERCEPTOR_ENTER(vname, str, __VA_ARGS__) \ @@ -1060,7 +1287,7 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_sscanf, __isoc99_vsscanf, str, format) // FIXME: under ASan the REAL() call below may write to freed memory and // corrupt its metadata. See -// https://code.google.com/p/address-sanitizer/issues/detail?id=321. +// https://github.com/google/sanitizers/issues/321. #define VSNPRINTF_INTERCEPTOR_IMPL(vname, str, size, ...) \ { \ VPRINTF_INTERCEPTOR_ENTER(vname, str, size, __VA_ARGS__) \ @@ -1077,7 +1304,7 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_sscanf, __isoc99_vsscanf, str, format) // FIXME: under ASan the REAL() call below may write to freed memory and // corrupt its metadata. See -// https://code.google.com/p/address-sanitizer/issues/detail?id=321. +// https://github.com/google/sanitizers/issues/321. #define VASPRINTF_INTERCEPTOR_IMPL(vname, strp, ...) \ { \ VPRINTF_INTERCEPTOR_ENTER(vname, strp, __VA_ARGS__) \ @@ -1362,7 +1589,7 @@ INTERCEPTOR(int, getpwnam_r, const char *name, __sanitizer_passwd *pwd, COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getpwnam_r)(name, pwd, buf, buflen, result); if (!res) { if (result && *result) unpoison_passwd(ctx, *result); @@ -1377,7 +1604,7 @@ INTERCEPTOR(int, getpwuid_r, u32 uid, __sanitizer_passwd *pwd, char *buf, COMMON_INTERCEPTOR_ENTER(ctx, getpwuid_r, uid, pwd, buf, buflen, result); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getpwuid_r)(uid, pwd, buf, buflen, result); if (!res) { if (result && *result) unpoison_passwd(ctx, *result); @@ -1393,7 +1620,7 @@ INTERCEPTOR(int, getgrnam_r, const char *name, __sanitizer_group *grp, COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getgrnam_r)(name, grp, buf, buflen, result); if (!res) { if (result && *result) unpoison_group(ctx, *result); @@ -1408,7 +1635,7 @@ INTERCEPTOR(int, getgrgid_r, u32 gid, __sanitizer_group *grp, char *buf, COMMON_INTERCEPTOR_ENTER(ctx, getgrgid_r, gid, grp, buf, buflen, result); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getgrgid_r)(gid, grp, buf, buflen, result); if (!res) { if (result && *result) unpoison_group(ctx, *result); @@ -1477,7 +1704,7 @@ INTERCEPTOR(int, getpwent_r, __sanitizer_passwd *pwbuf, char *buf, COMMON_INTERCEPTOR_ENTER(ctx, getpwent_r, pwbuf, buf, buflen, pwbufp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getpwent_r)(pwbuf, buf, buflen, pwbufp); if (!res) { if (pwbufp && *pwbufp) unpoison_passwd(ctx, *pwbufp); @@ -1492,7 +1719,7 @@ INTERCEPTOR(int, fgetpwent_r, void *fp, __sanitizer_passwd *pwbuf, char *buf, COMMON_INTERCEPTOR_ENTER(ctx, fgetpwent_r, fp, pwbuf, buf, buflen, pwbufp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(fgetpwent_r)(fp, pwbuf, buf, buflen, pwbufp); if (!res) { if (pwbufp && *pwbufp) unpoison_passwd(ctx, *pwbufp); @@ -1507,7 +1734,7 @@ INTERCEPTOR(int, getgrent_r, __sanitizer_group *pwbuf, char *buf, SIZE_T buflen, COMMON_INTERCEPTOR_ENTER(ctx, getgrent_r, pwbuf, buf, buflen, pwbufp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getgrent_r)(pwbuf, buf, buflen, pwbufp); if (!res) { if (pwbufp && *pwbufp) unpoison_group(ctx, *pwbufp); @@ -1522,7 +1749,7 @@ INTERCEPTOR(int, fgetgrent_r, void *fp, __sanitizer_group *pwbuf, char *buf, COMMON_INTERCEPTOR_ENTER(ctx, fgetgrent_r, fp, pwbuf, buf, buflen, pwbufp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(fgetgrent_r)(fp, pwbuf, buf, buflen, pwbufp); if (!res) { if (pwbufp && *pwbufp) unpoison_group(ctx, *pwbufp); @@ -1579,7 +1806,7 @@ INTERCEPTOR(int, clock_getres, u32 clk_id, void *tp) { COMMON_INTERCEPTOR_ENTER(ctx, clock_getres, clk_id, tp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(clock_getres)(clk_id, tp); if (!res && tp) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, tp, struct_timespec_sz); @@ -1591,7 +1818,7 @@ INTERCEPTOR(int, clock_gettime, u32 clk_id, void *tp) { COMMON_INTERCEPTOR_ENTER(ctx, clock_gettime, clk_id, tp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(clock_gettime)(clk_id, tp); if (!res) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, tp, struct_timespec_sz); @@ -1618,7 +1845,7 @@ INTERCEPTOR(int, getitimer, int which, void *curr_value) { COMMON_INTERCEPTOR_ENTER(ctx, getitimer, which, curr_value); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getitimer)(which, curr_value); if (!res && curr_value) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, curr_value, struct_itimerval_sz); @@ -1632,7 +1859,7 @@ INTERCEPTOR(int, setitimer, int which, const void *new_value, void *old_value) { COMMON_INTERCEPTOR_READ_RANGE(ctx, new_value, struct_itimerval_sz); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(setitimer)(which, new_value, old_value); if (!res && old_value) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, old_value, struct_itimerval_sz); @@ -1689,16 +1916,19 @@ static int wrapped_gl_stat(const char *s, void *st) { return pglob_copy->gl_stat(s, st); } +static const __sanitizer_glob_t kGlobCopy = { + 0, 0, 0, + 0, wrapped_gl_closedir, wrapped_gl_readdir, + wrapped_gl_opendir, wrapped_gl_lstat, wrapped_gl_stat}; + INTERCEPTOR(int, glob, const char *pattern, int flags, int (*errfunc)(const char *epath, int eerrno), __sanitizer_glob_t *pglob) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, glob, pattern, flags, errfunc, pglob); COMMON_INTERCEPTOR_READ_STRING(ctx, pattern, 0); - __sanitizer_glob_t glob_copy = { - 0, 0, 0, - 0, wrapped_gl_closedir, wrapped_gl_readdir, - wrapped_gl_opendir, wrapped_gl_lstat, wrapped_gl_stat}; + __sanitizer_glob_t glob_copy; + internal_memcpy(&glob_copy, &kGlobCopy, sizeof(glob_copy)); if (flags & glob_altdirfunc) { Swap(pglob->gl_closedir, glob_copy.gl_closedir); Swap(pglob->gl_readdir, glob_copy.gl_readdir); @@ -1726,10 +1956,8 @@ INTERCEPTOR(int, glob64, const char *pattern, int flags, void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, glob64, pattern, flags, errfunc, pglob); COMMON_INTERCEPTOR_READ_STRING(ctx, pattern, 0); - __sanitizer_glob_t glob_copy = { - 0, 0, 0, - 0, wrapped_gl_closedir, wrapped_gl_readdir, - wrapped_gl_opendir, wrapped_gl_lstat, wrapped_gl_stat}; + __sanitizer_glob_t glob_copy; + internal_memcpy(&glob_copy, &kGlobCopy, sizeof(glob_copy)); if (flags & glob_altdirfunc) { Swap(pglob->gl_closedir, glob_copy.gl_closedir); Swap(pglob->gl_readdir, glob_copy.gl_readdir); @@ -1766,7 +1994,7 @@ INTERCEPTOR_WITH_SUFFIX(int, wait, int *status) { COMMON_INTERCEPTOR_ENTER(ctx, wait, status); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(wait)(status); if (res != -1 && status) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status)); @@ -1784,7 +2012,7 @@ INTERCEPTOR_WITH_SUFFIX(int, waitid, int idtype, int id, void *infop, COMMON_INTERCEPTOR_ENTER(ctx, waitid, idtype, id, infop, options); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(waitid)(idtype, id, infop, options); if (res != -1 && infop) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, infop, siginfo_t_sz); @@ -1795,7 +2023,7 @@ INTERCEPTOR_WITH_SUFFIX(int, waitpid, int pid, int *status, int options) { COMMON_INTERCEPTOR_ENTER(ctx, waitpid, pid, status, options); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(waitpid)(pid, status, options); if (res != -1 && status) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status)); @@ -1806,7 +2034,7 @@ INTERCEPTOR(int, wait3, int *status, int options, void *rusage) { COMMON_INTERCEPTOR_ENTER(ctx, wait3, status, options, rusage); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(wait3)(status, options, rusage); if (res != -1) { if (status) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status)); @@ -1820,7 +2048,7 @@ INTERCEPTOR(int, __wait4, int pid, int *status, int options, void *rusage) { COMMON_INTERCEPTOR_ENTER(ctx, __wait4, pid, status, options, rusage); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(__wait4)(pid, status, options, rusage); if (res != -1) { if (status) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status)); @@ -1835,7 +2063,7 @@ INTERCEPTOR(int, wait4, int pid, int *status, int options, void *rusage) { COMMON_INTERCEPTOR_ENTER(ctx, wait4, pid, status, options, rusage); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(wait4)(pid, status, options, rusage); if (res != -1) { if (status) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status)); @@ -1864,7 +2092,7 @@ INTERCEPTOR(char *, inet_ntop, int af, const void *src, char *dst, u32 size) { // FIXME: figure out read size based on the address family. // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(inet_ntop)(af, src, dst, size); if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1); return res; @@ -1876,7 +2104,7 @@ INTERCEPTOR(int, inet_pton, int af, const char *src, void *dst) { // FIXME: figure out read size based on the address family. // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(inet_pton)(af, src, dst); if (res == 1) { uptr sz = __sanitizer_in_addr_sz(af); @@ -1898,7 +2126,7 @@ INTERCEPTOR(int, inet_aton, const char *cp, void *dst) { if (cp) COMMON_INTERCEPTOR_READ_RANGE(ctx, cp, REAL(strlen)(cp) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(inet_aton)(cp, dst); if (res != 0) { uptr sz = __sanitizer_in_addr_sz(af_inet); @@ -1917,7 +2145,7 @@ INTERCEPTOR(int, pthread_getschedparam, uptr thread, int *policy, int *param) { COMMON_INTERCEPTOR_ENTER(ctx, pthread_getschedparam, thread, policy, param); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(pthread_getschedparam)(thread, policy, param); if (res == 0) { if (policy) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, policy, sizeof(*policy)); @@ -1944,7 +2172,7 @@ INTERCEPTOR(int, getaddrinfo, char *node, char *service, COMMON_INTERCEPTOR_READ_RANGE(ctx, hints, sizeof(__sanitizer_addrinfo)); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getaddrinfo)(node, service, hints, out); if (res == 0 && out) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, out, sizeof(*out)); @@ -1976,7 +2204,7 @@ INTERCEPTOR(int, getnameinfo, void *sockaddr, unsigned salen, char *host, // There is padding in in_addr that may make this too noisy // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getnameinfo)(sockaddr, salen, host, hostlen, serv, servlen, flags); if (res == 0) { @@ -2000,7 +2228,7 @@ INTERCEPTOR(int, getsockname, int sock_fd, void *addr, int *addrlen) { int addrlen_in = *addrlen; // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getsockname)(sock_fd, addr, addrlen); if (res == 0) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, Min(addrlen_in, *addrlen)); @@ -2086,7 +2314,7 @@ INTERCEPTOR(int, gethostbyname_r, char *name, struct __sanitizer_hostent *ret, h_errnop); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(gethostbyname_r)(name, ret, buf, buflen, result, h_errnop); if (result) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); @@ -2109,7 +2337,7 @@ INTERCEPTOR(int, gethostent_r, struct __sanitizer_hostent *ret, char *buf, h_errnop); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(gethostent_r)(ret, buf, buflen, result, h_errnop); if (result) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); @@ -2135,7 +2363,7 @@ INTERCEPTOR(int, gethostbyaddr_r, void *addr, int len, int type, COMMON_INTERCEPTOR_READ_RANGE(ctx, addr, len); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(gethostbyaddr_r)(addr, len, type, ret, buf, buflen, result, h_errnop); if (result) { @@ -2161,7 +2389,7 @@ INTERCEPTOR(int, gethostbyname2_r, char *name, int af, result, h_errnop); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(gethostbyname2_r)(name, af, ret, buf, buflen, result, h_errnop); if (result) { @@ -2187,7 +2415,7 @@ INTERCEPTOR(int, getsockopt, int sockfd, int level, int optname, void *optval, if (optlen) COMMON_INTERCEPTOR_READ_RANGE(ctx, optlen, sizeof(*optlen)); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getsockopt)(sockfd, level, optname, optval, optlen); if (res == 0) if (optval && optlen) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, optval, *optlen); @@ -2231,7 +2459,7 @@ INTERCEPTOR(int, accept4, int fd, void *addr, unsigned *addrlen, int f) { } // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int fd2 = REAL(accept4)(fd, addr, addrlen, f); if (fd2 >= 0) { if (fd >= 0) COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, fd2); @@ -2251,7 +2479,7 @@ INTERCEPTOR(double, modf, double x, double *iptr) { COMMON_INTERCEPTOR_ENTER(ctx, modf, x, iptr); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. double res = REAL(modf)(x, iptr); if (iptr) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, iptr, sizeof(*iptr)); @@ -2263,7 +2491,7 @@ INTERCEPTOR(float, modff, float x, float *iptr) { COMMON_INTERCEPTOR_ENTER(ctx, modff, x, iptr); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. float res = REAL(modff)(x, iptr); if (iptr) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, iptr, sizeof(*iptr)); @@ -2275,7 +2503,7 @@ INTERCEPTOR(long double, modfl, long double x, long double *iptr) { COMMON_INTERCEPTOR_ENTER(ctx, modfl, x, iptr); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. long double res = REAL(modfl)(x, iptr); if (iptr) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, iptr, sizeof(*iptr)); @@ -2285,7 +2513,7 @@ INTERCEPTOR(long double, modfl, long double x, long double *iptr) { #define INIT_MODF \ COMMON_INTERCEPT_FUNCTION(modf); \ COMMON_INTERCEPT_FUNCTION(modff); \ - COMMON_INTERCEPT_FUNCTION(modfl); + COMMON_INTERCEPT_FUNCTION_LDBL(modfl); #else #define INIT_MODF #endif @@ -2310,7 +2538,7 @@ INTERCEPTOR(SSIZE_T, recvmsg, int fd, struct __sanitizer_msghdr *msg, COMMON_INTERCEPTOR_ENTER(ctx, recvmsg, fd, msg, flags); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SSIZE_T res = REAL(recvmsg)(fd, msg, flags); if (res >= 0) { if (fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd); @@ -2326,6 +2554,75 @@ INTERCEPTOR(SSIZE_T, recvmsg, int fd, struct __sanitizer_msghdr *msg, #define INIT_RECVMSG #endif +#if SANITIZER_INTERCEPT_SENDMSG +static void read_msghdr_control(void *ctx, void *control, uptr controllen) { + const unsigned kCmsgDataOffset = + RoundUpTo(sizeof(__sanitizer_cmsghdr), sizeof(uptr)); + + char *p = (char *)control; + char *const control_end = p + controllen; + while (true) { + if (p + sizeof(__sanitizer_cmsghdr) > control_end) break; + __sanitizer_cmsghdr *cmsg = (__sanitizer_cmsghdr *)p; + COMMON_INTERCEPTOR_READ_RANGE(ctx, &cmsg->cmsg_len, sizeof(cmsg->cmsg_len)); + + if (p + RoundUpTo(cmsg->cmsg_len, sizeof(uptr)) > control_end) break; + + COMMON_INTERCEPTOR_READ_RANGE(ctx, &cmsg->cmsg_level, + sizeof(cmsg->cmsg_level)); + COMMON_INTERCEPTOR_READ_RANGE(ctx, &cmsg->cmsg_type, + sizeof(cmsg->cmsg_type)); + + if (cmsg->cmsg_len > kCmsgDataOffset) { + char *data = p + kCmsgDataOffset; + unsigned data_len = cmsg->cmsg_len - kCmsgDataOffset; + if (data_len > 0) COMMON_INTERCEPTOR_READ_RANGE(ctx, data, data_len); + } + + p += RoundUpTo(cmsg->cmsg_len, sizeof(uptr)); + } +} + +static void read_msghdr(void *ctx, struct __sanitizer_msghdr *msg, + SSIZE_T maxlen) { +#define R(f) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, &msg->msg_##f, sizeof(msg->msg_##f)) + R(name); + R(namelen); + R(iov); + R(iovlen); + R(control); + R(controllen); + R(flags); +#undef R + if (msg->msg_name && msg->msg_namelen) + COMMON_INTERCEPTOR_READ_RANGE(ctx, msg->msg_name, msg->msg_namelen); + if (msg->msg_iov && msg->msg_iovlen) + COMMON_INTERCEPTOR_READ_RANGE(ctx, msg->msg_iov, + sizeof(*msg->msg_iov) * msg->msg_iovlen); + read_iovec(ctx, msg->msg_iov, msg->msg_iovlen, maxlen); + if (msg->msg_control && msg->msg_controllen) + read_msghdr_control(ctx, msg->msg_control, msg->msg_controllen); +} + +INTERCEPTOR(SSIZE_T, sendmsg, int fd, struct __sanitizer_msghdr *msg, + int flags) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, sendmsg, fd, msg, flags); + if (fd >= 0) { + COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); + COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd); + } + SSIZE_T res = REAL(sendmsg)(fd, msg, flags); + if (common_flags()->intercept_send && res >= 0 && msg) + read_msghdr(ctx, msg, res); + return res; +} +#define INIT_SENDMSG COMMON_INTERCEPT_FUNCTION(sendmsg); +#else +#define INIT_SENDMSG +#endif + #if SANITIZER_INTERCEPT_GETPEERNAME INTERCEPTOR(int, getpeername, int sockfd, void *addr, unsigned *addrlen) { void *ctx; @@ -2334,7 +2631,7 @@ INTERCEPTOR(int, getpeername, int sockfd, void *addr, unsigned *addrlen) { if (addrlen) addr_sz = *addrlen; // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getpeername)(sockfd, addr, addrlen); if (!res && addr && addrlen) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, Min(addr_sz, *addrlen)); @@ -2350,7 +2647,7 @@ INTERCEPTOR(int, sysinfo, void *info) { void *ctx; // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. COMMON_INTERCEPTOR_ENTER(ctx, sysinfo, info); int res = REAL(sysinfo)(info); if (!res && info) @@ -2378,7 +2675,7 @@ INTERCEPTOR(__sanitizer_dirent *, readdir, void *dirp) { COMMON_INTERCEPTOR_ENTER(ctx, readdir, dirp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. __sanitizer_dirent *res = REAL(readdir)(dirp); if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, res->d_reclen); return res; @@ -2390,7 +2687,7 @@ INTERCEPTOR(int, readdir_r, void *dirp, __sanitizer_dirent *entry, COMMON_INTERCEPTOR_ENTER(ctx, readdir_r, dirp, entry, result); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(readdir_r)(dirp, entry, result); if (!res) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); @@ -2414,7 +2711,7 @@ INTERCEPTOR(__sanitizer_dirent64 *, readdir64, void *dirp) { COMMON_INTERCEPTOR_ENTER(ctx, readdir64, dirp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. __sanitizer_dirent64 *res = REAL(readdir64)(dirp); if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, res->d_reclen); return res; @@ -2426,7 +2723,7 @@ INTERCEPTOR(int, readdir64_r, void *dirp, __sanitizer_dirent64 *entry, COMMON_INTERCEPTOR_ENTER(ctx, readdir64_r, dirp, entry, result); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(readdir64_r)(dirp, entry, result); if (!res) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); @@ -2473,7 +2770,7 @@ INTERCEPTOR(uptr, ptrace, int request, int pid, void *addr, void *data) { // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. uptr res = REAL(ptrace)(request, pid, addr, data); if (!res && data) { @@ -2528,7 +2825,7 @@ INTERCEPTOR(char *, getcwd, char *buf, SIZE_T size) { COMMON_INTERCEPTOR_ENTER(ctx, getcwd, buf, size); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(getcwd)(buf, size); if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1); return res; @@ -2544,7 +2841,7 @@ INTERCEPTOR(char *, get_current_dir_name, int fake) { COMMON_INTERCEPTOR_ENTER(ctx, get_current_dir_name, fake); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(get_current_dir_name)(fake); if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1); return res; @@ -2593,7 +2890,7 @@ INTERCEPTOR(INTMAX_T, strtoimax, const char *nptr, char **endptr, int base) { COMMON_INTERCEPTOR_ENTER(ctx, strtoimax, nptr, endptr, base); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *real_endptr; INTMAX_T res = REAL(strtoimax)(nptr, &real_endptr, base); StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base); @@ -2605,7 +2902,7 @@ INTERCEPTOR(INTMAX_T, strtoumax, const char *nptr, char **endptr, int base) { COMMON_INTERCEPTOR_ENTER(ctx, strtoumax, nptr, endptr, base); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *real_endptr; INTMAX_T res = REAL(strtoumax)(nptr, &real_endptr, base); StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base); @@ -2625,7 +2922,7 @@ INTERCEPTOR(SIZE_T, mbstowcs, wchar_t *dest, const char *src, SIZE_T len) { COMMON_INTERCEPTOR_ENTER(ctx, mbstowcs, dest, src, len); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SIZE_T res = REAL(mbstowcs)(dest, src, len); if (res != (SIZE_T) - 1 && dest) { SIZE_T write_cnt = res + (res < len); @@ -2642,7 +2939,7 @@ INTERCEPTOR(SIZE_T, mbsrtowcs, wchar_t *dest, const char **src, SIZE_T len, if (ps) COMMON_INTERCEPTOR_READ_RANGE(ctx, ps, mbstate_t_sz); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SIZE_T res = REAL(mbsrtowcs)(dest, src, len, ps); if (res != (SIZE_T)(-1) && dest && src) { // This function, and several others, may or may not write the terminating @@ -2672,7 +2969,7 @@ INTERCEPTOR(SIZE_T, mbsnrtowcs, wchar_t *dest, const char **src, SIZE_T nms, if (ps) COMMON_INTERCEPTOR_READ_RANGE(ctx, ps, mbstate_t_sz); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SIZE_T res = REAL(mbsnrtowcs)(dest, src, nms, len, ps); if (res != (SIZE_T)(-1) && dest && src) { SIZE_T write_cnt = res + !*src; @@ -2692,7 +2989,7 @@ INTERCEPTOR(SIZE_T, wcstombs, char *dest, const wchar_t *src, SIZE_T len) { COMMON_INTERCEPTOR_ENTER(ctx, wcstombs, dest, src, len); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SIZE_T res = REAL(wcstombs)(dest, src, len); if (res != (SIZE_T) - 1 && dest) { SIZE_T write_cnt = res + (res < len); @@ -2709,7 +3006,7 @@ INTERCEPTOR(SIZE_T, wcsrtombs, char *dest, const wchar_t **src, SIZE_T len, if (ps) COMMON_INTERCEPTOR_READ_RANGE(ctx, ps, mbstate_t_sz); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SIZE_T res = REAL(wcsrtombs)(dest, src, len, ps); if (res != (SIZE_T) - 1 && dest && src) { SIZE_T write_cnt = res + !*src; @@ -2737,7 +3034,7 @@ INTERCEPTOR(SIZE_T, wcsnrtombs, char *dest, const wchar_t **src, SIZE_T nms, if (ps) COMMON_INTERCEPTOR_READ_RANGE(ctx, ps, mbstate_t_sz); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SIZE_T res = REAL(wcsnrtombs)(dest, src, nms, len, ps); if (res != ((SIZE_T)-1) && dest && src) { SIZE_T write_cnt = res + !*src; @@ -2759,7 +3056,7 @@ INTERCEPTOR(SIZE_T, wcrtomb, char *dest, wchar_t src, void *ps) { if (ps) COMMON_INTERCEPTOR_READ_RANGE(ctx, ps, mbstate_t_sz); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SIZE_T res = REAL(wcrtomb)(dest, src, ps); if (res != ((SIZE_T)-1) && dest) { SIZE_T write_cnt = res; @@ -2779,7 +3076,7 @@ INTERCEPTOR(int, tcgetattr, int fd, void *termios_p) { COMMON_INTERCEPTOR_ENTER(ctx, tcgetattr, fd, termios_p); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(tcgetattr)(fd, termios_p); if (!res && termios_p) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, termios_p, struct_termios_sz); @@ -2836,7 +3133,7 @@ INTERCEPTOR(SIZE_T, confstr, int name, char *buf, SIZE_T len) { COMMON_INTERCEPTOR_ENTER(ctx, confstr, name, buf, len); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SIZE_T res = REAL(confstr)(name, buf, len); if (buf && res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, res < len ? res : len); @@ -2853,7 +3150,7 @@ INTERCEPTOR(int, sched_getaffinity, int pid, SIZE_T cpusetsize, void *mask) { COMMON_INTERCEPTOR_ENTER(ctx, sched_getaffinity, pid, cpusetsize, mask); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(sched_getaffinity)(pid, cpusetsize, mask); if (mask && !res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mask, cpusetsize); return res; @@ -2895,7 +3192,7 @@ INTERCEPTOR(char *, strerror_r, int errnum, char *buf, SIZE_T buflen) { COMMON_INTERCEPTOR_ENTER(ctx, strerror_r, errnum, buf, buflen); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(strerror_r)(errnum, buf, buflen); // There are 2 versions of strerror_r: // * POSIX version returns 0 on success, negative error code on failure, @@ -2926,7 +3223,7 @@ INTERCEPTOR(int, __xpg_strerror_r, int errnum, char *buf, SIZE_T buflen) { COMMON_INTERCEPTOR_ENTER(ctx, __xpg_strerror_r, errnum, buf, buflen); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(__xpg_strerror_r)(errnum, buf, buflen); // This version always returns a null-terminated string. if (buf && buflen) @@ -2971,7 +3268,7 @@ INTERCEPTOR(int, scandir, char *dirp, __sanitizer_dirent ***namelist, scandir_compar = compar; // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(scandir)(dirp, namelist, filter ? wrapped_scandir_filter : nullptr, compar ? wrapped_scandir_compar : nullptr); @@ -3024,7 +3321,7 @@ INTERCEPTOR(int, scandir64, char *dirp, __sanitizer_dirent64 ***namelist, scandir64_compar = compar; // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(scandir64)(dirp, namelist, filter ? wrapped_scandir64_filter : nullptr, @@ -3051,7 +3348,7 @@ INTERCEPTOR(int, getgroups, int size, u32 *lst) { COMMON_INTERCEPTOR_ENTER(ctx, getgroups, size, lst); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getgroups)(size, lst); if (res && lst) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, lst, res * sizeof(*lst)); return res; @@ -3117,7 +3414,7 @@ INTERCEPTOR(int, wordexp, char *s, __sanitizer_wordexp_t *p, int flags) { if (s) COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(wordexp)(s, p, flags); if (!res && p) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p)); @@ -3143,7 +3440,7 @@ INTERCEPTOR(int, sigwait, __sanitizer_sigset_t *set, int *sig) { // FIXME: read sigset_t when all of sigemptyset, etc are intercepted // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(sigwait)(set, sig); if (!res && sig) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sig, sizeof(*sig)); return res; @@ -3160,7 +3457,7 @@ INTERCEPTOR(int, sigwaitinfo, __sanitizer_sigset_t *set, void *info) { // FIXME: read sigset_t when all of sigemptyset, etc are intercepted // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(sigwaitinfo)(set, info); if (res > 0 && info) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, info, siginfo_t_sz); return res; @@ -3179,7 +3476,7 @@ INTERCEPTOR(int, sigtimedwait, __sanitizer_sigset_t *set, void *info, // FIXME: read sigset_t when all of sigemptyset, etc are intercepted // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(sigtimedwait)(set, info, timeout); if (res > 0 && info) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, info, siginfo_t_sz); return res; @@ -3195,7 +3492,7 @@ INTERCEPTOR(int, sigemptyset, __sanitizer_sigset_t *set) { COMMON_INTERCEPTOR_ENTER(ctx, sigemptyset, set); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(sigemptyset)(set); if (!res && set) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, set, sizeof(*set)); return res; @@ -3206,7 +3503,7 @@ INTERCEPTOR(int, sigfillset, __sanitizer_sigset_t *set) { COMMON_INTERCEPTOR_ENTER(ctx, sigfillset, set); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(sigfillset)(set); if (!res && set) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, set, sizeof(*set)); return res; @@ -3224,7 +3521,7 @@ INTERCEPTOR(int, sigpending, __sanitizer_sigset_t *set) { COMMON_INTERCEPTOR_ENTER(ctx, sigpending, set); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(sigpending)(set); if (!res && set) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, set, sizeof(*set)); return res; @@ -3242,7 +3539,7 @@ INTERCEPTOR(int, sigprocmask, int how, __sanitizer_sigset_t *set, // FIXME: read sigset_t when all of sigemptyset, etc are intercepted // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(sigprocmask)(how, set, oldset); if (!res && oldset) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldset, sizeof(*oldset)); @@ -3259,7 +3556,7 @@ INTERCEPTOR(int, backtrace, void **buffer, int size) { COMMON_INTERCEPTOR_ENTER(ctx, backtrace, buffer, size); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(backtrace)(buffer, size); if (res && buffer) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buffer, res * sizeof(*buffer)); @@ -3273,7 +3570,7 @@ INTERCEPTOR(char **, backtrace_symbols, void **buffer, int size) { COMMON_INTERCEPTOR_READ_RANGE(ctx, buffer, size * sizeof(*buffer)); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char **res = REAL(backtrace_symbols)(buffer, size); if (res && size) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, size * sizeof(*res)); @@ -3293,7 +3590,9 @@ INTERCEPTOR(char **, backtrace_symbols, void **buffer, int size) { INTERCEPTOR(void, _exit, int status) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, _exit, status); + COMMON_INTERCEPTOR_USER_CALLBACK_START(); int status1 = COMMON_INTERCEPTOR_ON_EXIT(ctx); + COMMON_INTERCEPTOR_USER_CALLBACK_END(); if (status == 0) status = status1; REAL(_exit)(status); } @@ -3311,6 +3610,8 @@ INTERCEPTOR(int, pthread_mutex_lock, void *m) { COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m); if (res == 0 || res == errno_EOWNERDEAD) COMMON_INTERCEPTOR_MUTEX_LOCK(ctx, m); + if (res == errno_EINVAL) + COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m); return res; } @@ -3318,7 +3619,10 @@ INTERCEPTOR(int, pthread_mutex_unlock, void *m) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, pthread_mutex_unlock, m); COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m); - return REAL(pthread_mutex_unlock)(m); + int res = REAL(pthread_mutex_unlock)(m); + if (res == errno_EINVAL) + COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m); + return res; } #define INIT_PTHREAD_MUTEX_LOCK COMMON_INTERCEPT_FUNCTION(pthread_mutex_lock) @@ -3381,7 +3685,7 @@ INTERCEPTOR(int, statfs, char *path, void *buf) { if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(statfs)(path, buf); if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statfs_sz); return res; @@ -3391,7 +3695,7 @@ INTERCEPTOR(int, fstatfs, int fd, void *buf) { COMMON_INTERCEPTOR_ENTER(ctx, fstatfs, fd, buf); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(fstatfs)(fd, buf); if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statfs_sz); return res; @@ -3410,7 +3714,7 @@ INTERCEPTOR(int, statfs64, char *path, void *buf) { if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(statfs64)(path, buf); if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statfs64_sz); return res; @@ -3420,7 +3724,7 @@ INTERCEPTOR(int, fstatfs64, int fd, void *buf) { COMMON_INTERCEPTOR_ENTER(ctx, fstatfs64, fd, buf); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(fstatfs64)(fd, buf); if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statfs64_sz); return res; @@ -3439,7 +3743,7 @@ INTERCEPTOR(int, statvfs, char *path, void *buf) { if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(statvfs)(path, buf); if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs_sz); return res; @@ -3449,7 +3753,7 @@ INTERCEPTOR(int, fstatvfs, int fd, void *buf) { COMMON_INTERCEPTOR_ENTER(ctx, fstatvfs, fd, buf); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(fstatvfs)(fd, buf); if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs_sz); return res; @@ -3468,7 +3772,7 @@ INTERCEPTOR(int, statvfs64, char *path, void *buf) { if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(statvfs64)(path, buf); if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs64_sz); return res; @@ -3478,7 +3782,7 @@ INTERCEPTOR(int, fstatvfs64, int fd, void *buf) { COMMON_INTERCEPTOR_ENTER(ctx, fstatvfs64, fd, buf); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(fstatvfs64)(fd, buf); if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs64_sz); return res; @@ -3534,7 +3838,7 @@ INTERCEPTOR(int, ether_ntohost, char *hostname, __sanitizer_ether_addr *addr) { if (addr) COMMON_INTERCEPTOR_READ_RANGE(ctx, addr, sizeof(*addr)); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(ether_ntohost)(hostname, addr); if (!res && hostname) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, hostname, REAL(strlen)(hostname) + 1); @@ -3547,7 +3851,7 @@ INTERCEPTOR(int, ether_hostton, char *hostname, __sanitizer_ether_addr *addr) { COMMON_INTERCEPTOR_READ_RANGE(ctx, hostname, REAL(strlen)(hostname) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(ether_hostton)(hostname, addr); if (!res && addr) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, sizeof(*addr)); return res; @@ -3559,7 +3863,7 @@ INTERCEPTOR(int, ether_line, char *line, __sanitizer_ether_addr *addr, if (line) COMMON_INTERCEPTOR_READ_RANGE(ctx, line, REAL(strlen)(line) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(ether_line)(line, addr, hostname); if (!res) { if (addr) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, sizeof(*addr)); @@ -3583,7 +3887,7 @@ INTERCEPTOR(char *, ether_ntoa_r, __sanitizer_ether_addr *addr, char *buf) { if (addr) COMMON_INTERCEPTOR_READ_RANGE(ctx, addr, sizeof(*addr)); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(ether_ntoa_r)(addr, buf); if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1); return res; @@ -3595,7 +3899,7 @@ INTERCEPTOR(__sanitizer_ether_addr *, ether_aton_r, char *buf, if (buf) COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, REAL(strlen)(buf) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. __sanitizer_ether_addr *res = REAL(ether_aton_r)(buf, addr); if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, sizeof(*res)); return res; @@ -3613,7 +3917,7 @@ INTERCEPTOR(int, shmctl, int shmid, int cmd, void *buf) { COMMON_INTERCEPTOR_ENTER(ctx, shmctl, shmid, cmd, buf); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(shmctl)(shmid, cmd, buf); if (res >= 0) { unsigned sz = 0; @@ -3638,7 +3942,7 @@ INTERCEPTOR(int, random_r, void *buf, u32 *result) { COMMON_INTERCEPTOR_ENTER(ctx, random_r, buf, result); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(random_r)(buf, result); if (!res && result) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); @@ -3651,7 +3955,7 @@ INTERCEPTOR(int, random_r, void *buf, u32 *result) { // FIXME: under ASan the REAL() call below may write to freed memory and corrupt // its metadata. See -// https://code.google.com/p/address-sanitizer/issues/detail?id=321. +// https://github.com/google/sanitizers/issues/321. #if SANITIZER_INTERCEPT_PTHREAD_ATTR_GET || \ SANITIZER_INTERCEPT_PTHREAD_ATTR_GETINHERITSSCHED || \ SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GET || \ @@ -3690,7 +3994,7 @@ INTERCEPTOR(int, pthread_attr_getstack, void *attr, void **addr, SIZE_T *size) { COMMON_INTERCEPTOR_ENTER(ctx, pthread_attr_getstack, attr, addr, size); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(pthread_attr_getstack)(attr, addr, size); if (!res) { if (addr) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, sizeof(*addr)); @@ -3739,7 +4043,7 @@ INTERCEPTOR(int, pthread_attr_getaffinity_np, void *attr, SIZE_T cpusetsize, cpuset); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(pthread_attr_getaffinity_np)(attr, cpusetsize, cpuset); if (!res && cpusetsize && cpuset) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cpuset, cpusetsize); @@ -3849,7 +4153,7 @@ INTERCEPTOR(char *, tmpnam, char *s) { if (s) // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, REAL(strlen)(s) + 1); else COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1); @@ -3867,7 +4171,7 @@ INTERCEPTOR(char *, tmpnam_r, char *s) { COMMON_INTERCEPTOR_ENTER(ctx, tmpnam_r, s); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(tmpnam_r)(s); if (res && s) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, REAL(strlen)(s) + 1); return res; @@ -3877,6 +4181,20 @@ INTERCEPTOR(char *, tmpnam_r, char *s) { #define INIT_TMPNAM_R #endif +#if SANITIZER_INTERCEPT_TTYNAME_R +INTERCEPTOR(int, ttyname_r, int fd, char *name, SIZE_T namesize) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, ttyname_r, fd, name, namesize); + int res = REAL(ttyname_r)(fd, name, namesize); + if (res == 0) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, REAL(strlen)(name) + 1); + return res; +} +#define INIT_TTYNAME_R COMMON_INTERCEPT_FUNCTION(ttyname_r); +#else +#define INIT_TTYNAME_R +#endif + #if SANITIZER_INTERCEPT_TEMPNAM INTERCEPTOR(char *, tempnam, char *dir, char *pfx) { void *ctx; @@ -3911,7 +4229,7 @@ INTERCEPTOR(void, sincos, double x, double *sin, double *cos) { COMMON_INTERCEPTOR_ENTER(ctx, sincos, x, sin, cos); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. REAL(sincos)(x, sin, cos); if (sin) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sin, sizeof(*sin)); if (cos) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cos, sizeof(*cos)); @@ -3921,7 +4239,7 @@ INTERCEPTOR(void, sincosf, float x, float *sin, float *cos) { COMMON_INTERCEPTOR_ENTER(ctx, sincosf, x, sin, cos); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. REAL(sincosf)(x, sin, cos); if (sin) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sin, sizeof(*sin)); if (cos) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cos, sizeof(*cos)); @@ -3931,7 +4249,7 @@ INTERCEPTOR(void, sincosl, long double x, long double *sin, long double *cos) { COMMON_INTERCEPTOR_ENTER(ctx, sincosl, x, sin, cos); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. REAL(sincosl)(x, sin, cos); if (sin) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sin, sizeof(*sin)); if (cos) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cos, sizeof(*cos)); @@ -3939,7 +4257,7 @@ INTERCEPTOR(void, sincosl, long double x, long double *sin, long double *cos) { #define INIT_SINCOS \ COMMON_INTERCEPT_FUNCTION(sincos); \ COMMON_INTERCEPT_FUNCTION(sincosf); \ - COMMON_INTERCEPT_FUNCTION(sincosl); + COMMON_INTERCEPT_FUNCTION_LDBL(sincosl); #else #define INIT_SINCOS #endif @@ -3950,7 +4268,7 @@ INTERCEPTOR(double, remquo, double x, double y, int *quo) { COMMON_INTERCEPTOR_ENTER(ctx, remquo, x, y, quo); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. double res = REAL(remquo)(x, y, quo); if (quo) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, quo, sizeof(*quo)); return res; @@ -3960,7 +4278,7 @@ INTERCEPTOR(float, remquof, float x, float y, int *quo) { COMMON_INTERCEPTOR_ENTER(ctx, remquof, x, y, quo); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. float res = REAL(remquof)(x, y, quo); if (quo) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, quo, sizeof(*quo)); return res; @@ -3970,7 +4288,7 @@ INTERCEPTOR(long double, remquol, long double x, long double y, int *quo) { COMMON_INTERCEPTOR_ENTER(ctx, remquol, x, y, quo); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. long double res = REAL(remquol)(x, y, quo); if (quo) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, quo, sizeof(*quo)); return res; @@ -3978,7 +4296,7 @@ INTERCEPTOR(long double, remquol, long double x, long double y, int *quo) { #define INIT_REMQUO \ COMMON_INTERCEPT_FUNCTION(remquo); \ COMMON_INTERCEPT_FUNCTION(remquof); \ - COMMON_INTERCEPT_FUNCTION(remquol); + COMMON_INTERCEPT_FUNCTION_LDBL(remquol); #else #define INIT_REMQUO #endif @@ -4009,7 +4327,7 @@ INTERCEPTOR(long double, lgammal, long double x) { #define INIT_LGAMMA \ COMMON_INTERCEPT_FUNCTION(lgamma); \ COMMON_INTERCEPT_FUNCTION(lgammaf); \ - COMMON_INTERCEPT_FUNCTION(lgammal); + COMMON_INTERCEPT_FUNCTION_LDBL(lgammal); #else #define INIT_LGAMMA #endif @@ -4020,7 +4338,7 @@ INTERCEPTOR(double, lgamma_r, double x, int *signp) { COMMON_INTERCEPTOR_ENTER(ctx, lgamma_r, x, signp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. double res = REAL(lgamma_r)(x, signp); if (signp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, signp, sizeof(*signp)); return res; @@ -4030,7 +4348,7 @@ INTERCEPTOR(float, lgammaf_r, float x, int *signp) { COMMON_INTERCEPTOR_ENTER(ctx, lgammaf_r, x, signp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. float res = REAL(lgammaf_r)(x, signp); if (signp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, signp, sizeof(*signp)); return res; @@ -4048,12 +4366,12 @@ INTERCEPTOR(long double, lgammal_r, long double x, int *signp) { COMMON_INTERCEPTOR_ENTER(ctx, lgammal_r, x, signp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. long double res = REAL(lgammal_r)(x, signp); if (signp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, signp, sizeof(*signp)); return res; } -#define INIT_LGAMMAL_R COMMON_INTERCEPT_FUNCTION(lgammal_r); +#define INIT_LGAMMAL_R COMMON_INTERCEPT_FUNCTION_LDBL(lgammal_r); #else #define INIT_LGAMMAL_R #endif @@ -4064,7 +4382,7 @@ INTERCEPTOR(int, drand48_r, void *buffer, double *result) { COMMON_INTERCEPTOR_ENTER(ctx, drand48_r, buffer, result); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(drand48_r)(buffer, result); if (result) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); return res; @@ -4074,7 +4392,7 @@ INTERCEPTOR(int, lrand48_r, void *buffer, long *result) { COMMON_INTERCEPTOR_ENTER(ctx, lrand48_r, buffer, result); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(lrand48_r)(buffer, result); if (result) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); return res; @@ -4104,7 +4422,7 @@ INTERCEPTOR(SSIZE_T, getline, char **lineptr, SIZE_T *n, void *stream) { COMMON_INTERCEPTOR_ENTER(ctx, getline, lineptr, n, stream); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SSIZE_T res = REAL(getline)(lineptr, n, stream); if (res > 0) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, lineptr, sizeof(*lineptr)); @@ -4116,7 +4434,7 @@ INTERCEPTOR(SSIZE_T, getline, char **lineptr, SIZE_T *n, void *stream) { // FIXME: under ASan the call below may write to freed memory and corrupt its // metadata. See -// https://code.google.com/p/address-sanitizer/issues/detail?id=321. +// https://github.com/google/sanitizers/issues/321. #define GETDELIM_INTERCEPTOR_IMPL(vname) \ { \ void *ctx; \ @@ -4163,7 +4481,7 @@ INTERCEPTOR(SIZE_T, iconv, void *cd, char **inbuf, SIZE_T *inbytesleft, void *outbuf_orig = outbuf ? *outbuf : nullptr; // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SIZE_T res = REAL(iconv)(cd, inbuf, inbytesleft, outbuf, outbytesleft); if (res != (SIZE_T) - 1 && outbuf && *outbuf > outbuf_orig) { SIZE_T sz = (char *)*outbuf - (char *)outbuf_orig; @@ -4182,7 +4500,7 @@ INTERCEPTOR(__sanitizer_clock_t, times, void *tms) { COMMON_INTERCEPTOR_ENTER(ctx, times, tms); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. __sanitizer_clock_t res = REAL(times)(tms); if (res != (__sanitizer_clock_t)-1 && tms) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, tms, struct_tms_sz); @@ -4194,6 +4512,7 @@ INTERCEPTOR(__sanitizer_clock_t, times, void *tms) { #endif #if SANITIZER_INTERCEPT_TLS_GET_ADDR +#if !SANITIZER_S390 #define INIT_TLS_GET_ADDR COMMON_INTERCEPT_FUNCTION(__tls_get_addr) // If you see any crashes around this functions, there are 2 known issues with // it: 1. __tls_get_addr can be called with mis-aligned stack due to: @@ -4214,6 +4533,67 @@ INTERCEPTOR(void *, __tls_get_addr, void *arg) { } return res; } +#if SANITIZER_PPC +// On PowerPC, we also need to intercept __tls_get_addr_opt, which has +// mostly the same semantics as __tls_get_addr, but its presence enables +// some optimizations in linker (which are safe to ignore here). +extern "C" __attribute__((alias("__interceptor___tls_get_addr"), + visibility("default"))) +void *__tls_get_addr_opt(void *arg); +#endif +#else // SANITIZER_S390 +// On s390, we have to intercept two functions here: +// - __tls_get_addr_internal, which is a glibc-internal function that is like +// the usual __tls_get_addr, but returns a TP-relative offset instead of +// a proper pointer. It is used by dlsym for TLS symbols. +// - __tls_get_offset, which is like the above, but also takes a GOT-relative +// descriptor offset as an argument instead of a pointer. GOT address +// is passed in r12, so it's necessary to write it in assembly. This is +// the function used by the compiler. +#define INIT_TLS_GET_ADDR COMMON_INTERCEPT_FUNCTION(__tls_get_addr_internal) +INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, __tls_get_addr_internal, arg); + uptr res = REAL(__tls_get_addr_internal)(arg); + uptr tp = reinterpret_cast<uptr>(__builtin_thread_pointer()); + void *ptr = reinterpret_cast<void *>(res + tp); + uptr tls_begin, tls_end; + COMMON_INTERCEPTOR_GET_TLS_RANGE(&tls_begin, &tls_end); + DTLS::DTV *dtv = DTLS_on_tls_get_addr(arg, ptr, tls_begin, tls_end); + if (dtv) { + // New DTLS block has been allocated. + COMMON_INTERCEPTOR_INITIALIZE_RANGE((void *)dtv->beg, dtv->size); + } + return res; +} +// We need a protected symbol aliasing the above, so that we can jump +// directly to it from the assembly below. +extern "C" __attribute__((alias("__interceptor___tls_get_addr_internal"), + visibility("protected"))) +uptr __interceptor___tls_get_addr_internal_protected(void *arg); +// Now carefully intercept __tls_get_offset. +asm( + ".text\n" + ".global __tls_get_offset\n" + "__tls_get_offset:\n" +// The __intercept_ version has to exist, so that gen_dynamic_list.py +// exports our symbol. + ".global __interceptor___tls_get_offset\n" + "__interceptor___tls_get_offset:\n" +#ifdef __s390x__ + "la %r2, 0(%r2,%r12)\n" + "jg __interceptor___tls_get_addr_internal_protected\n" +#else + "basr %r3,0\n" + "0: la %r2,0(%r2,%r12)\n" + "l %r4,1f-0b(%r3)\n" + "b 0(%r4,%r3)\n" + "1: .long __interceptor___tls_get_addr_internal_protected - 0b\n" +#endif + ".type __tls_get_offset, @function\n" + ".size __tls_get_offset, .-__tls_get_offset\n" +); +#endif // SANITIZER_S390 #else #define INIT_TLS_GET_ADDR #endif @@ -4225,7 +4605,7 @@ INTERCEPTOR(SSIZE_T, listxattr, const char *path, char *list, SIZE_T size) { if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SSIZE_T res = REAL(listxattr)(path, list, size); // Here and below, size == 0 is a special case where nothing is written to the // buffer, and res contains the desired buffer size. @@ -4238,7 +4618,7 @@ INTERCEPTOR(SSIZE_T, llistxattr, const char *path, char *list, SIZE_T size) { if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SSIZE_T res = REAL(llistxattr)(path, list, size); if (size && res > 0 && list) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, list, res); return res; @@ -4248,7 +4628,7 @@ INTERCEPTOR(SSIZE_T, flistxattr, int fd, char *list, SIZE_T size) { COMMON_INTERCEPTOR_ENTER(ctx, flistxattr, fd, list, size); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SSIZE_T res = REAL(flistxattr)(fd, list, size); if (size && res > 0 && list) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, list, res); return res; @@ -4270,7 +4650,7 @@ INTERCEPTOR(SSIZE_T, getxattr, const char *path, const char *name, char *value, if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SSIZE_T res = REAL(getxattr)(path, name, value, size); if (size && res > 0 && value) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, value, res); return res; @@ -4283,7 +4663,7 @@ INTERCEPTOR(SSIZE_T, lgetxattr, const char *path, const char *name, char *value, if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SSIZE_T res = REAL(lgetxattr)(path, name, value, size); if (size && res > 0 && value) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, value, res); return res; @@ -4295,7 +4675,7 @@ INTERCEPTOR(SSIZE_T, fgetxattr, int fd, const char *name, char *value, if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SSIZE_T res = REAL(fgetxattr)(fd, name, value, size); if (size && res > 0 && value) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, value, res); return res; @@ -4314,7 +4694,7 @@ INTERCEPTOR(int, getresuid, void *ruid, void *euid, void *suid) { COMMON_INTERCEPTOR_ENTER(ctx, getresuid, ruid, euid, suid); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getresuid)(ruid, euid, suid); if (res >= 0) { if (ruid) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ruid, uid_t_sz); @@ -4328,7 +4708,7 @@ INTERCEPTOR(int, getresgid, void *rgid, void *egid, void *sgid) { COMMON_INTERCEPTOR_ENTER(ctx, getresgid, rgid, egid, sgid); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getresgid)(rgid, egid, sgid); if (res >= 0) { if (rgid) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rgid, gid_t_sz); @@ -4353,7 +4733,7 @@ INTERCEPTOR(int, getifaddrs, __sanitizer_ifaddrs **ifap) { COMMON_INTERCEPTOR_ENTER(ctx, getifaddrs, ifap); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getifaddrs)(ifap); if (res == 0 && ifap) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ifap, sizeof(void *)); @@ -4389,7 +4769,7 @@ INTERCEPTOR(char *, if_indextoname, unsigned int ifindex, char* ifname) { COMMON_INTERCEPTOR_ENTER(ctx, if_indextoname, ifindex, ifname); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(if_indextoname)(ifindex, ifname); if (res && ifname) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ifname, REAL(strlen)(ifname) + 1); @@ -4417,7 +4797,7 @@ INTERCEPTOR(int, capget, void *hdrp, void *datap) { COMMON_INTERCEPTOR_READ_RANGE(ctx, hdrp, __user_cap_header_struct_sz); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(capget)(hdrp, datap); if (res == 0 && datap) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, datap, __user_cap_data_struct_sz); @@ -4518,7 +4898,7 @@ INTERCEPTOR(int, ftime, __sanitizer_timeb *tp) { COMMON_INTERCEPTOR_ENTER(ctx, ftime, tp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(ftime)(tp); if (tp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, tp, sizeof(*tp)); @@ -4536,7 +4916,7 @@ INTERCEPTOR(void, xdrmem_create, __sanitizer_XDR *xdrs, uptr addr, COMMON_INTERCEPTOR_ENTER(ctx, xdrmem_create, xdrs, addr, size, op); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. REAL(xdrmem_create)(xdrs, addr, size, op); COMMON_INTERCEPTOR_WRITE_RANGE(ctx, xdrs, sizeof(*xdrs)); if (op == __sanitizer_XDR_ENCODE) { @@ -4551,14 +4931,14 @@ INTERCEPTOR(void, xdrstdio_create, __sanitizer_XDR *xdrs, void *file, int op) { COMMON_INTERCEPTOR_ENTER(ctx, xdrstdio_create, xdrs, file, op); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. REAL(xdrstdio_create)(xdrs, file, op); COMMON_INTERCEPTOR_WRITE_RANGE(ctx, xdrs, sizeof(*xdrs)); } // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See -// https://code.google.com/p/address-sanitizer/issues/detail?id=321. +// https://github.com/google/sanitizers/issues/321. #define XDR_INTERCEPTOR(F, T) \ INTERCEPTOR(int, F, __sanitizer_XDR *xdrs, T *p) { \ void *ctx; \ @@ -4612,7 +4992,7 @@ INTERCEPTOR(int, xdr_bytes, __sanitizer_XDR *xdrs, char **p, unsigned *sizep, } // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(xdr_bytes)(xdrs, p, sizep, maxsize); if (p && sizep && xdrs->x_op == __sanitizer_XDR_DECODE) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p)); @@ -4632,7 +5012,7 @@ INTERCEPTOR(int, xdr_string, __sanitizer_XDR *xdrs, char **p, } // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(xdr_string)(xdrs, p, maxsize); if (p && xdrs->x_op == __sanitizer_XDR_DECODE) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p)); @@ -4684,7 +5064,7 @@ INTERCEPTOR(void *, tsearch, void *key, void **rootp, COMMON_INTERCEPTOR_ENTER(ctx, tsearch, key, rootp, compar); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. void *res = REAL(tsearch)(key, rootp, compar); if (res && *(void **)res == key) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, sizeof(void *)); @@ -4766,7 +5146,7 @@ INTERCEPTOR(int, __woverflow, __sanitizer_FILE *fp, int ch) { INTERCEPTOR(__sanitizer_FILE *, fopen, const char *path, const char *mode) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, fopen, path, mode); - COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); + if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1); __sanitizer_FILE *res = REAL(fopen)(path, mode); COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, path); @@ -4837,7 +5217,7 @@ INTERCEPTOR(__sanitizer_FILE *, open_memstream, char **ptr, SIZE_T *sizeloc) { COMMON_INTERCEPTOR_ENTER(ctx, open_memstream, ptr, sizeloc); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. __sanitizer_FILE *res = REAL(open_memstream)(ptr, sizeloc); if (res) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, sizeof(*ptr)); @@ -4868,7 +5248,7 @@ INTERCEPTOR(__sanitizer_FILE *, fmemopen, void *buf, SIZE_T size, COMMON_INTERCEPTOR_ENTER(ctx, fmemopen, buf, size, mode); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. __sanitizer_FILE *res = REAL(fmemopen)(buf, size, mode); if (res) unpoison_file(res); return res; @@ -5282,22 +5662,245 @@ INTERCEPTOR(SSIZE_T, process_vm_writev, int pid, __sanitizer_iovec *local_iov, #define INIT_PROCESS_VM_READV #endif +#if SANITIZER_INTERCEPT_CTERMID +INTERCEPTOR(char *, ctermid, char *s) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, ctermid, s); + char *res = REAL(ctermid)(s); + if (res) { + COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1); + } + return res; +} +#define INIT_CTERMID COMMON_INTERCEPT_FUNCTION(ctermid); +#else +#define INIT_CTERMID +#endif + +#if SANITIZER_INTERCEPT_CTERMID_R +INTERCEPTOR(char *, ctermid_r, char *s) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, ctermid_r, s); + char *res = REAL(ctermid_r)(s); + if (res) { + COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1); + } + return res; +} +#define INIT_CTERMID_R COMMON_INTERCEPT_FUNCTION(ctermid_r); +#else +#define INIT_CTERMID_R +#endif + +#if SANITIZER_INTERCEPT_RECV_RECVFROM +INTERCEPTOR(SSIZE_T, recv, int fd, void *buf, SIZE_T len, int flags) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, recv, fd, buf, len, flags); + COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); + SSIZE_T res = REAL(recv)(fd, buf, len, flags); + if (res > 0) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, Min((SIZE_T)res, len)); + } + if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd); + return res; +} + +INTERCEPTOR(SSIZE_T, recvfrom, int fd, void *buf, SIZE_T len, int flags, + void *srcaddr, int *addrlen) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, recvfrom, fd, buf, len, flags, srcaddr, + addrlen); + COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); + SIZE_T srcaddr_sz; + if (srcaddr) srcaddr_sz = *addrlen; + (void)srcaddr_sz; // prevent "set but not used" warning + SSIZE_T res = REAL(recvfrom)(fd, buf, len, flags, srcaddr, addrlen); + if (res > 0) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, Min((SIZE_T)res, len)); + if (srcaddr) + COMMON_INTERCEPTOR_INITIALIZE_RANGE(srcaddr, + Min((SIZE_T)*addrlen, srcaddr_sz)); + } + return res; +} +#define INIT_RECV_RECVFROM \ + COMMON_INTERCEPT_FUNCTION(recv); \ + COMMON_INTERCEPT_FUNCTION(recvfrom); +#else +#define INIT_RECV_RECVFROM +#endif + +#if SANITIZER_INTERCEPT_SEND_SENDTO +INTERCEPTOR(SSIZE_T, send, int fd, void *buf, SIZE_T len, int flags) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, send, fd, buf, len, flags); + if (fd >= 0) { + COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); + COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd); + } + SSIZE_T res = REAL(send)(fd, buf, len, flags); + if (common_flags()->intercept_send && res > 0) + COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, Min((SIZE_T)res, len)); + return res; +} + +INTERCEPTOR(SSIZE_T, sendto, int fd, void *buf, SIZE_T len, int flags, + void *dstaddr, int addrlen) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, sendto, fd, buf, len, flags, dstaddr, addrlen); + if (fd >= 0) { + COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); + COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd); + } + // Can't check dstaddr as it may have uninitialized padding at the end. + SSIZE_T res = REAL(sendto)(fd, buf, len, flags, dstaddr, addrlen); + if (common_flags()->intercept_send && res > 0) + COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, Min((SIZE_T)res, len)); + return res; +} +#define INIT_SEND_SENDTO \ + COMMON_INTERCEPT_FUNCTION(send); \ + COMMON_INTERCEPT_FUNCTION(sendto); +#else +#define INIT_SEND_SENDTO +#endif + +#if SANITIZER_INTERCEPT_EVENTFD_READ_WRITE +INTERCEPTOR(int, eventfd_read, int fd, u64 *value) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, eventfd_read, fd, value); + COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); + int res = REAL(eventfd_read)(fd, value); + if (res == 0) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, value, sizeof(*value)); + if (fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd); + } + return res; +} +INTERCEPTOR(int, eventfd_write, int fd, u64 value) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, eventfd_write, fd, value); + if (fd >= 0) { + COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); + COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd); + } + int res = REAL(eventfd_write)(fd, value); + return res; +} +#define INIT_EVENTFD_READ_WRITE \ + COMMON_INTERCEPT_FUNCTION(eventfd_read); \ + COMMON_INTERCEPT_FUNCTION(eventfd_write) +#else +#define INIT_EVENTFD_READ_WRITE +#endif + +#if SANITIZER_INTERCEPT_STAT +INTERCEPTOR(int, stat, const char *path, void *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, stat, path, buf); + if (common_flags()->intercept_stat) + COMMON_INTERCEPTOR_READ_STRING(ctx, path, 0); + int res = REAL(stat)(path, buf); + if (!res) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer::struct_stat_sz); + return res; +} +#define INIT_STAT COMMON_INTERCEPT_FUNCTION(stat) +#else +#define INIT_STAT +#endif + +#if SANITIZER_INTERCEPT___XSTAT +INTERCEPTOR(int, __xstat, int version, const char *path, void *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, __xstat, version, path, buf); + if (common_flags()->intercept_stat) + COMMON_INTERCEPTOR_READ_STRING(ctx, path, 0); + int res = REAL(__xstat)(version, path, buf); + if (!res) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer::struct_stat_sz); + return res; +} +#define INIT___XSTAT COMMON_INTERCEPT_FUNCTION(__xstat) +#else +#define INIT___XSTAT +#endif + +#if SANITIZER_INTERCEPT___XSTAT64 +INTERCEPTOR(int, __xstat64, int version, const char *path, void *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, __xstat64, version, path, buf); + if (common_flags()->intercept_stat) + COMMON_INTERCEPTOR_READ_STRING(ctx, path, 0); + int res = REAL(__xstat64)(version, path, buf); + if (!res) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer::struct_stat64_sz); + return res; +} +#define INIT___XSTAT64 COMMON_INTERCEPT_FUNCTION(__xstat64) +#else +#define INIT___XSTAT64 +#endif + +#if SANITIZER_INTERCEPT___LXSTAT +INTERCEPTOR(int, __lxstat, int version, const char *path, void *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, __lxstat, version, path, buf); + if (common_flags()->intercept_stat) + COMMON_INTERCEPTOR_READ_STRING(ctx, path, 0); + int res = REAL(__lxstat)(version, path, buf); + if (!res) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer::struct_stat_sz); + return res; +} +#define INIT___LXSTAT COMMON_INTERCEPT_FUNCTION(__lxstat) +#else +#define INIT___LXSTAT +#endif + +#if SANITIZER_INTERCEPT___LXSTAT64 +INTERCEPTOR(int, __lxstat64, int version, const char *path, void *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, __lxstat64, version, path, buf); + if (common_flags()->intercept_stat) + COMMON_INTERCEPTOR_READ_STRING(ctx, path, 0); + int res = REAL(__lxstat64)(version, path, buf); + if (!res) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer::struct_stat64_sz); + return res; +} +#define INIT___LXSTAT64 COMMON_INTERCEPT_FUNCTION(__lxstat64) +#else +#define INIT___LXSTAT64 +#endif + +// FIXME: add other *stat interceptor + static void InitializeCommonInterceptors() { static u64 metadata_mem[sizeof(MetadataHashMap) / sizeof(u64) + 1]; interceptor_metadata_map = new((void *)&metadata_mem) MetadataHashMap(); INIT_TEXTDOMAIN; + INIT_STRLEN; + INIT_STRNLEN; INIT_STRCMP; INIT_STRNCMP; INIT_STRCASECMP; INIT_STRNCASECMP; INIT_STRSTR; INIT_STRCASESTR; + INIT_STRCHR; + INIT_STRCHRNUL; + INIT_STRRCHR; INIT_STRSPN; INIT_STRPBRK; + INIT_MEMSET; + INIT_MEMMOVE; + INIT_MEMCPY; INIT_MEMCHR; INIT_MEMCMP; INIT_MEMRCHR; + INIT_MEMMEM; INIT_READ; INIT_PREAD; INIT_PREAD64; @@ -5347,6 +5950,7 @@ static void InitializeCommonInterceptors() { INIT_ACCEPT4; INIT_MODF; INIT_RECVMSG; + INIT_SENDMSG; INIT_GETPEERNAME; INIT_IOCTL; INIT_INET_ATON; @@ -5416,6 +6020,7 @@ static void InitializeCommonInterceptors() { INIT_PTHREAD_BARRIERATTR_GETPSHARED; INIT_TMPNAM; INIT_TMPNAM_R; + INIT_TTYNAME_R; INIT_TEMPNAM; INIT_PTHREAD_SETNAME_NP; INIT_SINCOS; @@ -5456,4 +6061,15 @@ static void InitializeCommonInterceptors() { INIT_PTHREAD_SETCANCEL; INIT_MINCORE; INIT_PROCESS_VM_READV; + INIT_CTERMID; + INIT_CTERMID_R; + INIT_RECV_RECVFROM; + INIT_SEND_SENDTO; + INIT_STAT; + INIT_EVENTFD_READ_WRITE; + INIT___XSTAT; + INIT___XSTAT64; + INIT___LXSTAT; + INIT___LXSTAT64; + // FIXME: add other *stat interceptors. } diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interceptors_ioctl.inc b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_ioctl.inc index 6c5fda09fbf3..a68534c5a0a9 100755 --- a/libsanitizer/sanitizer_common/sanitizer_common_interceptors_ioctl.inc +++ b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_ioctl.inc @@ -51,25 +51,9 @@ static void ioctl_table_fill() { _(FIONBIO, READ, sizeof(int)); _(FIONCLEX, NONE, 0); _(FIOSETOWN, READ, sizeof(int)); - _(SIOCADDMULTI, READ, struct_ifreq_sz); _(SIOCATMARK, WRITE, sizeof(int)); - _(SIOCDELMULTI, READ, struct_ifreq_sz); - _(SIOCGIFADDR, WRITE, struct_ifreq_sz); - _(SIOCGIFBRDADDR, WRITE, struct_ifreq_sz); _(SIOCGIFCONF, CUSTOM, 0); - _(SIOCGIFDSTADDR, WRITE, struct_ifreq_sz); - _(SIOCGIFFLAGS, WRITE, struct_ifreq_sz); - _(SIOCGIFMETRIC, WRITE, struct_ifreq_sz); - _(SIOCGIFMTU, WRITE, struct_ifreq_sz); - _(SIOCGIFNETMASK, WRITE, struct_ifreq_sz); _(SIOCGPGRP, WRITE, sizeof(int)); - _(SIOCSIFADDR, READ, struct_ifreq_sz); - _(SIOCSIFBRDADDR, READ, struct_ifreq_sz); - _(SIOCSIFDSTADDR, READ, struct_ifreq_sz); - _(SIOCSIFFLAGS, READ, struct_ifreq_sz); - _(SIOCSIFMETRIC, READ, struct_ifreq_sz); - _(SIOCSIFMTU, READ, struct_ifreq_sz); - _(SIOCSIFNETMASK, READ, struct_ifreq_sz); _(SIOCSPGRP, READ, sizeof(int)); _(TIOCCONS, NONE, 0); _(TIOCEXCL, NONE, 0); @@ -90,6 +74,25 @@ static void ioctl_table_fill() { _(TIOCSTI, READ, sizeof(char)); _(TIOCSWINSZ, READ, struct_winsize_sz); +#if !SANITIZER_IOS + _(SIOCADDMULTI, READ, struct_ifreq_sz); + _(SIOCDELMULTI, READ, struct_ifreq_sz); + _(SIOCGIFADDR, WRITE, struct_ifreq_sz); + _(SIOCGIFBRDADDR, WRITE, struct_ifreq_sz); + _(SIOCGIFDSTADDR, WRITE, struct_ifreq_sz); + _(SIOCGIFFLAGS, WRITE, struct_ifreq_sz); + _(SIOCGIFMETRIC, WRITE, struct_ifreq_sz); + _(SIOCGIFMTU, WRITE, struct_ifreq_sz); + _(SIOCGIFNETMASK, WRITE, struct_ifreq_sz); + _(SIOCSIFADDR, READ, struct_ifreq_sz); + _(SIOCSIFBRDADDR, READ, struct_ifreq_sz); + _(SIOCSIFDSTADDR, READ, struct_ifreq_sz); + _(SIOCSIFFLAGS, READ, struct_ifreq_sz); + _(SIOCSIFMETRIC, READ, struct_ifreq_sz); + _(SIOCSIFMTU, READ, struct_ifreq_sz); + _(SIOCSIFNETMASK, READ, struct_ifreq_sz); +#endif + #if (SANITIZER_LINUX && !SANITIZER_ANDROID) _(SIOCGETSGCNT, WRITE, struct_sioc_sg_req_sz); _(SIOCGETVIFCNT, WRITE, struct_sioc_vif_req_sz); @@ -578,7 +581,8 @@ static void ioctl_common_pre(void *ctx, const ioctl_desc *desc, int d, return; if (request == IOCTL_SIOCGIFCONF) { struct __sanitizer_ifconf *ifc = (__sanitizer_ifconf *)arg; - COMMON_INTERCEPTOR_READ_RANGE(ctx, &ifc->ifc_len, sizeof(ifc->ifc_len)); + COMMON_INTERCEPTOR_READ_RANGE(ctx, (char*)&ifc->ifc_len, + sizeof(ifc->ifc_len)); } } diff --git a/libsanitizer/sanitizer_common/sanitizer_common_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_common_libcdep.cc index 5a76c4ebd8bf..8c9fa98f835f 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common_libcdep.cc +++ b/libsanitizer/sanitizer_common/sanitizer_common_libcdep.cc @@ -10,6 +10,8 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common.h" + +#include "sanitizer_allocator_interface.h" #include "sanitizer_flags.h" #include "sanitizer_stackdepot.h" #include "sanitizer_stacktrace.h" @@ -43,7 +45,8 @@ void SetSandboxingCallback(void (*f)()) { sandboxing_callback = f; } -void ReportErrorSummary(const char *error_type, StackTrace *stack) { +void ReportErrorSummary(const char *error_type, const StackTrace *stack) { +#if !SANITIZER_GO if (!common_flags()->print_summary) return; if (stack->size == 0) { @@ -56,6 +59,7 @@ void ReportErrorSummary(const char *error_type, StackTrace *stack) { SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc); ReportErrorSummary(error_type, frame->info); frame->ClearAll(); +#endif } static void (*SoftRssLimitExceededCallback)(bool exceeded); @@ -64,12 +68,22 @@ void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded)) { SoftRssLimitExceededCallback = Callback; } +static AllocatorReleaseToOSCallback ReleseCallback; +void SetAllocatorReleaseToOSCallback(AllocatorReleaseToOSCallback Callback) { + CHECK_EQ(ReleseCallback, nullptr); + ReleseCallback = Callback; +} + +#if SANITIZER_LINUX && !SANITIZER_GO void BackgroundThread(void *arg) { uptr hard_rss_limit_mb = common_flags()->hard_rss_limit_mb; uptr soft_rss_limit_mb = common_flags()->soft_rss_limit_mb; + bool heap_profile = common_flags()->heap_profile; + bool allocator_release_to_os = common_flags()->allocator_release_to_os; uptr prev_reported_rss = 0; uptr prev_reported_stack_depot_size = 0; bool reached_soft_rss_limit = false; + uptr rss_during_last_reported_profile = 0; while (true) { SleepForMillis(100); uptr current_rss_mb = GetRSS() >> 20; @@ -111,14 +125,43 @@ void BackgroundThread(void *arg) { SoftRssLimitExceededCallback(false); } } + if (allocator_release_to_os && ReleseCallback) ReleseCallback(); + if (heap_profile && + current_rss_mb > rss_during_last_reported_profile * 1.1) { + Printf("\n\nHEAP PROFILE at RSS %zdMb\n", current_rss_mb); + __sanitizer_print_memory_profile(90); + rss_during_last_reported_profile = current_rss_mb; + } } } +#endif + +void WriteToSyslog(const char *msg) { + InternalScopedString msg_copy(kErrorMessageBufferSize); + msg_copy.append("%s", msg); + char *p = msg_copy.data(); + char *q; + + // Print one line at a time. + // syslog, at least on Android, has an implicit message length limit. + do { + q = internal_strchr(p, '\n'); + if (q) + *q = '\0'; + WriteOneLineToSyslog(p); + if (q) + p = q + 1; + } while (q); +} void MaybeStartBackgroudThread() { -#if SANITIZER_LINUX // Need to implement/test on other platforms. +#if SANITIZER_LINUX && \ + !SANITIZER_GO // Need to implement/test on other platforms. // Start the background thread if one of the rss limits is given. if (!common_flags()->hard_rss_limit_mb && - !common_flags()->soft_rss_limit_mb) return; + !common_flags()->soft_rss_limit_mb && + !common_flags()->allocator_release_to_os && + !common_flags()->heap_profile) return; if (!&real_pthread_create) return; // Can't spawn the thread anyway. internal_start_thread(BackgroundThread, nullptr); #endif @@ -128,7 +171,7 @@ void MaybeStartBackgroudThread() { void NOINLINE __sanitizer_sandbox_on_notify(__sanitizer_sandbox_arguments *args) { - PrepareForSandboxing(args); - if (sandboxing_callback) - sandboxing_callback(); + __sanitizer::PrepareForSandboxing(args); + if (__sanitizer::sandboxing_callback) + __sanitizer::sandboxing_callback(); } diff --git a/libsanitizer/sanitizer_common/sanitizer_common_syscalls.inc b/libsanitizer/sanitizer_common/sanitizer_common_syscalls.inc index 5e83ce9786c9..6fd5ef74274c 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common_syscalls.inc +++ b/libsanitizer/sanitizer_common/sanitizer_common_syscalls.inc @@ -1235,17 +1235,15 @@ POST_SYSCALL(fcntl64)(long res, long fd, long cmd, long arg) {} PRE_SYSCALL(pipe)(void *fildes) {} POST_SYSCALL(pipe)(long res, void *fildes) { - if (res >= 0) { - if (fildes) POST_WRITE(fildes, sizeof(int)); - } + if (res >= 0) + if (fildes) POST_WRITE(fildes, sizeof(int) * 2); } PRE_SYSCALL(pipe2)(void *fildes, long flags) {} POST_SYSCALL(pipe2)(long res, void *fildes, long flags) { - if (res >= 0) { - if (fildes) POST_WRITE(fildes, sizeof(int)); - } + if (res >= 0) + if (fildes) POST_WRITE(fildes, sizeof(int) * 2); } PRE_SYSCALL(dup)(long fildes) {} @@ -1878,13 +1876,11 @@ PRE_SYSCALL(socket)(long arg0, long arg1, long arg2) {} POST_SYSCALL(socket)(long res, long arg0, long arg1, long arg2) {} -PRE_SYSCALL(socketpair)(long arg0, long arg1, long arg2, void *arg3) {} +PRE_SYSCALL(socketpair)(long arg0, long arg1, long arg2, int *sv) {} -POST_SYSCALL(socketpair)(long res, long arg0, long arg1, long arg2, - void *arg3) { - if (res >= 0) { - if (arg3) POST_WRITE(arg3, sizeof(int)); - } +POST_SYSCALL(socketpair)(long res, long arg0, long arg1, long arg2, int *sv) { + if (res >= 0) + if (sv) POST_WRITE(sv, sizeof(int) * 2); } PRE_SYSCALL(socketcall)(long call, void *args) {} @@ -2299,7 +2295,7 @@ POST_SYSCALL(ni_syscall)(long res) {} PRE_SYSCALL(ptrace)(long request, long pid, long addr, long data) { #if !SANITIZER_ANDROID && \ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \ - defined(__powerpc64__) || defined(__aarch64__)) + defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__)) if (data) { if (request == ptrace_setregs) { PRE_READ((void *)data, struct_user_regs_struct_sz); @@ -2320,7 +2316,7 @@ PRE_SYSCALL(ptrace)(long request, long pid, long addr, long data) { POST_SYSCALL(ptrace)(long res, long request, long pid, long addr, long data) { #if !SANITIZER_ANDROID && \ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \ - defined(__powerpc64__) || defined(__aarch64__)) + defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__)) if (res >= 0 && data) { // Note that this is different from the interceptor in // sanitizer_common_interceptors.inc. @@ -2842,6 +2838,40 @@ PRE_SYSCALL(vfork)() { POST_SYSCALL(vfork)(long res) { COMMON_SYSCALL_POST_FORK(res); } + +PRE_SYSCALL(sigaction)(long signum, const __sanitizer_kernel_sigaction_t *act, + __sanitizer_kernel_sigaction_t *oldact) { + if (act) { + PRE_READ(&act->sigaction, sizeof(act->sigaction)); + PRE_READ(&act->sa_flags, sizeof(act->sa_flags)); + PRE_READ(&act->sa_mask, sizeof(act->sa_mask)); + } +} + +POST_SYSCALL(sigaction)(long res, long signum, + const __sanitizer_kernel_sigaction_t *act, + __sanitizer_kernel_sigaction_t *oldact) { + if (res >= 0 && oldact) POST_WRITE(oldact, sizeof(*oldact)); +} + +PRE_SYSCALL(rt_sigaction)(long signum, + const __sanitizer_kernel_sigaction_t *act, + __sanitizer_kernel_sigaction_t *oldact, SIZE_T sz) { + if (act) { + PRE_READ(&act->sigaction, sizeof(act->sigaction)); + PRE_READ(&act->sa_flags, sizeof(act->sa_flags)); + PRE_READ(&act->sa_mask, sz); + } +} + +POST_SYSCALL(rt_sigaction)(long res, long signum, + const __sanitizer_kernel_sigaction_t *act, + __sanitizer_kernel_sigaction_t *oldact, SIZE_T sz) { + if (res >= 0 && oldact) { + SIZE_T oldact_sz = ((char *)&oldact->sa_mask) - ((char *)oldact) + sz; + POST_WRITE(oldact, oldact_sz); + } +} } // extern "C" #undef PRE_SYSCALL diff --git a/libsanitizer/sanitizer_common/sanitizer_coverage_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_coverage_libcdep.cc index c67880468b8c..dd8620beaac0 100644 --- a/libsanitizer/sanitizer_common/sanitizer_coverage_libcdep.cc +++ b/libsanitizer/sanitizer_common/sanitizer_coverage_libcdep.cc @@ -45,8 +45,12 @@ #include "sanitizer_symbolizer.h" #include "sanitizer_flags.h" +using namespace __sanitizer; + static const u64 kMagic64 = 0xC0BFFFFFFFFFFF64ULL; static const u64 kMagic32 = 0xC0BFFFFFFFFFFF32ULL; +static const uptr kNumWordsForMagic = SANITIZER_WORDSIZE == 64 ? 1 : 2; +static const u64 kMagic = SANITIZER_WORDSIZE == 64 ? kMagic64 : kMagic32; static atomic_uint32_t dump_once_guard; // Ensure that CovDump runs only once. @@ -94,7 +98,7 @@ class CoverageData { void DumpAll(); ALWAYS_INLINE - void TraceBasicBlock(s32 *id); + void TraceBasicBlock(u32 *id); void InitializeGuardArray(s32 *guards); void InitializeGuards(s32 *guards, uptr n, const char *module_name, @@ -105,17 +109,23 @@ class CoverageData { uptr Update8bitCounterBitsetAndClearCounters(u8 *bitset); uptr *data(); - uptr size(); + uptr size() const; private: + struct NamedPcRange { + const char *copied_module_name; + uptr beg, end; // elements [beg,end) in pc_array. + }; + void DirectOpen(); void UpdateModuleNameVec(uptr caller_pc, uptr range_beg, uptr range_end); + void GetRangeOffsets(const NamedPcRange& r, Symbolizer* s, + InternalMmapVector<uptr>* offsets) const; // Maximal size pc array may ever grow. // We MmapNoReserve this space to ensure that the array is contiguous. - static const uptr kPcArrayMaxSize = FIRST_32_SECOND_64( - 1 << (SANITIZER_ANDROID ? 24 : (SANITIZER_WINDOWS ? 27 : 26)), - 1 << 27); + static const uptr kPcArrayMaxSize = + FIRST_32_SECOND_64(1 << (SANITIZER_ANDROID ? 24 : 26), 1 << 27); // The amount file mapping for the pc array is grown by. static const uptr kPcArrayMmapSize = 64 * 1024; @@ -134,11 +144,6 @@ class CoverageData { // Vector of coverage guard arrays, protected by mu. InternalMmapVectorNoCtor<s32*> guard_array_vec; - struct NamedPcRange { - const char *copied_module_name; - uptr beg, end; // elements [beg,end) in pc_array. - }; - // Vector of module and compilation unit pc ranges. InternalMmapVectorNoCtor<NamedPcRange> comp_unit_name_vec; InternalMmapVectorNoCtor<NamedPcRange> module_name_vec; @@ -510,7 +515,7 @@ uptr *CoverageData::data() { return pc_array; } -uptr CoverageData::size() { +uptr CoverageData::size() const { return atomic_load(&pc_array_index, memory_order_relaxed); } @@ -680,11 +685,11 @@ void CoverageData::DumpCallerCalleePairs() { // it once and then cache in the provided 'cache' storage. // // This function will eventually be inlined by the compiler. -void CoverageData::TraceBasicBlock(s32 *id) { +void CoverageData::TraceBasicBlock(u32 *id) { // Will trap here if // 1. coverage is not enabled at run-time. // 2. The array tr_event_array is full. - *tr_event_pointer = static_cast<u32>(*id - 1); + *tr_event_pointer = *id - 1; tr_event_pointer++; } @@ -740,41 +745,96 @@ void CoverageData::DumpAsBitSet() { } } + +void CoverageData::GetRangeOffsets(const NamedPcRange& r, Symbolizer* sym, + InternalMmapVector<uptr>* offsets) const { + offsets->clear(); + for (uptr i = 0; i < kNumWordsForMagic; i++) + offsets->push_back(0); + CHECK(r.copied_module_name); + CHECK_LE(r.beg, r.end); + CHECK_LE(r.end, size()); + for (uptr i = r.beg; i < r.end; i++) { + uptr pc = UnbundlePc(pc_array[i]); + uptr counter = UnbundleCounter(pc_array[i]); + if (!pc) continue; // Not visited. + uptr offset = 0; + sym->GetModuleNameAndOffsetForPC(pc, nullptr, &offset); + offsets->push_back(BundlePcAndCounter(offset, counter)); + } + + CHECK_GE(offsets->size(), kNumWordsForMagic); + SortArray(offsets->data(), offsets->size()); + for (uptr i = 0; i < offsets->size(); i++) + (*offsets)[i] = UnbundlePc((*offsets)[i]); +} + +static void GenerateHtmlReport(const InternalMmapVector<char *> &cov_files) { + if (!common_flags()->html_cov_report) { + return; + } + char *sancov_path = FindPathToBinary(common_flags()->sancov_path); + if (sancov_path == nullptr) { + return; + } + + InternalMmapVector<char *> sancov_argv(cov_files.size() * 2 + 3); + sancov_argv.push_back(sancov_path); + sancov_argv.push_back(internal_strdup("-html-report")); + auto argv_deleter = at_scope_exit([&] { + for (uptr i = 0; i < sancov_argv.size(); ++i) { + InternalFree(sancov_argv[i]); + } + }); + + for (const auto &cov_file : cov_files) { + sancov_argv.push_back(internal_strdup(cov_file)); + } + + { + ListOfModules modules; + modules.init(); + for (const LoadedModule &module : modules) { + sancov_argv.push_back(internal_strdup(module.full_name())); + } + } + + InternalScopedString report_path(kMaxPathLength); + fd_t report_fd = + CovOpenFile(&report_path, false /* packed */, GetProcessName(), "html"); + int pid = StartSubprocess(sancov_argv[0], sancov_argv.data(), + kInvalidFd /* stdin */, report_fd /* std_out */); + if (pid > 0) { + int result = WaitForProcess(pid); + if (result == 0) + Printf("coverage report generated to %s\n", report_path.data()); + } +} + void CoverageData::DumpOffsets() { auto sym = Symbolizer::GetOrInit(); if (!common_flags()->coverage_pcs) return; CHECK_NE(sym, nullptr); InternalMmapVector<uptr> offsets(0); InternalScopedString path(kMaxPathLength); - for (uptr m = 0; m < module_name_vec.size(); m++) { - offsets.clear(); - uptr num_words_for_magic = SANITIZER_WORDSIZE == 64 ? 1 : 2; - for (uptr i = 0; i < num_words_for_magic; i++) - offsets.push_back(0); - auto r = module_name_vec[m]; - CHECK(r.copied_module_name); - CHECK_LE(r.beg, r.end); - CHECK_LE(r.end, size()); - for (uptr i = r.beg; i < r.end; i++) { - uptr pc = UnbundlePc(pc_array[i]); - uptr counter = UnbundleCounter(pc_array[i]); - if (!pc) continue; // Not visited. - uptr offset = 0; - sym->GetModuleNameAndOffsetForPC(pc, nullptr, &offset); - offsets.push_back(BundlePcAndCounter(offset, counter)); + + InternalMmapVector<char *> cov_files(module_name_vec.size()); + auto cov_files_deleter = at_scope_exit([&] { + for (uptr i = 0; i < cov_files.size(); ++i) { + InternalFree(cov_files[i]); } + }); - CHECK_GE(offsets.size(), num_words_for_magic); - SortArray(offsets.data(), offsets.size()); - for (uptr i = 0; i < offsets.size(); i++) - offsets[i] = UnbundlePc(offsets[i]); + for (uptr m = 0; m < module_name_vec.size(); m++) { + auto r = module_name_vec[m]; + GetRangeOffsets(r, sym, &offsets); - uptr num_offsets = offsets.size() - num_words_for_magic; + uptr num_offsets = offsets.size() - kNumWordsForMagic; u64 *magic_p = reinterpret_cast<u64*>(offsets.data()); CHECK_EQ(*magic_p, 0ULL); // FIXME: we may want to write 32-bit offsets even in 64-mode // if all the offsets are small enough. - *magic_p = SANITIZER_WORDSIZE == 64 ? kMagic64 : kMagic32; + *magic_p = kMagic; const char *module_name = StripModuleName(r.copied_module_name); if (cov_sandboxed) { @@ -789,11 +849,14 @@ void CoverageData::DumpOffsets() { if (fd == kInvalidFd) continue; WriteToFile(fd, offsets.data(), offsets.size() * sizeof(offsets[0])); CloseFile(fd); + cov_files.push_back(internal_strdup(path.data())); VReport(1, " CovDump: %s: %zd PCs written\n", path.data(), num_offsets); } } if (cov_fd != kInvalidFd) CloseFile(cov_fd); + + GenerateHtmlReport(cov_files); } void CoverageData::DumpAll() { @@ -918,11 +981,13 @@ uptr __sanitizer_get_total_unique_caller_callee_pairs() { } SANITIZER_INTERFACE_ATTRIBUTE -void __sanitizer_cov_trace_func_enter(s32 *id) { +void __sanitizer_cov_trace_func_enter(u32 *id) { + __sanitizer_cov_with_check(id); coverage_data.TraceBasicBlock(id); } SANITIZER_INTERFACE_ATTRIBUTE -void __sanitizer_cov_trace_basic_block(s32 *id) { +void __sanitizer_cov_trace_basic_block(u32 *id) { + __sanitizer_cov_with_check(id); coverage_data.TraceBasicBlock(id); } SANITIZER_INTERFACE_ATTRIBUTE @@ -949,8 +1014,30 @@ uptr __sanitizer_update_counter_bitset_and_clear_counters(u8 *bitset) { return coverage_data.Update8bitCounterBitsetAndClearCounters(bitset); } // Default empty implementations (weak). Users should redefine them. +#if !SANITIZER_WINDOWS // weak does not work on Windows. SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_cmp() {} SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +void __sanitizer_cov_trace_cmp1() {} +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +void __sanitizer_cov_trace_cmp2() {} +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +void __sanitizer_cov_trace_cmp4() {} +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +void __sanitizer_cov_trace_cmp8() {} +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_switch() {} +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +void __sanitizer_cov_trace_div4() {} +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +void __sanitizer_cov_trace_div8() {} +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +void __sanitizer_cov_trace_gep() {} +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +void __sanitizer_cov_trace_pc_guard() {} +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +void __sanitizer_cov_trace_pc_indir() {} +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +void __sanitizer_cov_trace_pc_guard_init() {} +#endif // !SANITIZER_WINDOWS } // extern "C" diff --git a/libsanitizer/sanitizer_common/sanitizer_coverage_mapping_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_coverage_mapping_libcdep.cc index ebac68128818..b2e724ab221a 100644 --- a/libsanitizer/sanitizer_common/sanitizer_coverage_mapping_libcdep.cc +++ b/libsanitizer/sanitizer_common/sanitizer_coverage_mapping_libcdep.cc @@ -70,26 +70,21 @@ void CovUpdateMapping(const char *coverage_dir, uptr caller_pc) { InternalScopedString text(kMaxTextSize); { - InternalScopedBuffer<LoadedModule> modules(kMaxNumberOfModules); - CHECK(modules.data()); - int n_modules = GetListOfModules(modules.data(), kMaxNumberOfModules, - /* filter */ nullptr); - text.append("%d\n", sizeof(uptr) * 8); - for (int i = 0; i < n_modules; ++i) { - const char *module_name = StripModuleName(modules[i].full_name()); - uptr base = modules[i].base_address(); - for (auto iter = modules[i].ranges(); iter.hasNext();) { - const auto *range = iter.next(); - if (range->executable) { - uptr start = range->beg; - uptr end = range->end; + ListOfModules modules; + modules.init(); + for (const LoadedModule &module : modules) { + const char *module_name = StripModuleName(module.full_name()); + uptr base = module.base_address(); + for (const auto &range : module.ranges()) { + if (range.executable) { + uptr start = range.beg; + uptr end = range.end; text.append("%zx %zx %zx %s\n", start, end, base, module_name); if (caller_pc && caller_pc >= start && caller_pc < end) cached_mapping.SetModuleRange(start, end); } } - modules[i].clear(); } } diff --git a/libsanitizer/sanitizer_common/sanitizer_deadlock_detector1.cc b/libsanitizer/sanitizer_common/sanitizer_deadlock_detector1.cc index 7a318c963f74..e2aedc24da9e 100644 --- a/libsanitizer/sanitizer_common/sanitizer_deadlock_detector1.cc +++ b/libsanitizer/sanitizer_common/sanitizer_deadlock_detector1.cc @@ -117,11 +117,16 @@ void DD::MutexBeforeLock(DDCallback *cb, void DD::ReportDeadlock(DDCallback *cb, DDMutex *m) { DDLogicalThread *lt = cb->lt; - uptr path[10]; + uptr path[20]; uptr len = dd.findPathToLock(<->dd, m->id, path, ARRAY_SIZE(path)); - CHECK_GT(len, 0U); // Hm.. cycle of 10 locks? I'd like to see that. + if (len == 0U) { + // A cycle of 20+ locks? Well, that's a bit odd... + Printf("WARNING: too long mutex cycle found\n"); + return; + } CHECK_EQ(m->id, path[0]); lt->report_pending = true; + len = Min<uptr>(len, DDReport::kMaxLoopSize); DDReport *rep = <->rep; rep->n = len; for (uptr i = 0; i < len; i++) { diff --git a/libsanitizer/sanitizer_common/sanitizer_deadlock_detector_interface.h b/libsanitizer/sanitizer_common/sanitizer_deadlock_detector_interface.h index 07c3755155f2..f8da20612dbd 100644 --- a/libsanitizer/sanitizer_common/sanitizer_deadlock_detector_interface.h +++ b/libsanitizer/sanitizer_common/sanitizer_deadlock_detector_interface.h @@ -49,7 +49,7 @@ struct DDFlags { }; struct DDReport { - enum { kMaxLoopSize = 8 }; + enum { kMaxLoopSize = 20 }; int n; // number of entries in loop struct { u64 thr_ctx; // user thread context diff --git a/libsanitizer/sanitizer_common/sanitizer_flags.cc b/libsanitizer/sanitizer_common/sanitizer_flags.cc index a24ff900011c..5f69e58ffb27 100644 --- a/libsanitizer/sanitizer_common/sanitizer_flags.cc +++ b/libsanitizer/sanitizer_common/sanitizer_flags.cc @@ -28,11 +28,6 @@ struct FlagDescription { IntrusiveList<FlagDescription> flag_descriptions; -// If set, the tool will install its own SEGV signal handler by default. -#ifndef SANITIZER_NEEDS_SEGV -# define SANITIZER_NEEDS_SEGV 1 -#endif - void CommonFlags::SetDefaults() { #define COMMON_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; #include "sanitizer_flags.inc" @@ -43,17 +38,44 @@ void CommonFlags::CopyFrom(const CommonFlags &other) { internal_memcpy(this, &other, sizeof(*this)); } -// Copy the string from "s" to "out", replacing "%b" with the binary basename. -static void SubstituteBinaryName(const char *s, char *out, uptr out_size) { +// Copy the string from "s" to "out", making the following substitutions: +// %b = binary basename +// %p = pid +void SubstituteForFlagValue(const char *s, char *out, uptr out_size) { char *out_end = out + out_size; while (*s && out < out_end - 1) { - if (s[0] != '%' || s[1] != 'b') { *out++ = *s++; continue; } - const char *base = GetProcessName(); - CHECK(base); - while (*base && out < out_end - 1) - *out++ = *base++; - s += 2; // skip "%b" + if (s[0] != '%') { + *out++ = *s++; + continue; + } + switch (s[1]) { + case 'b': { + const char *base = GetProcessName(); + CHECK(base); + while (*base && out < out_end - 1) + *out++ = *base++; + s += 2; // skip "%b" + break; + } + case 'p': { + int pid = internal_getpid(); + char buf[32]; + char *buf_pos = buf + 32; + do { + *--buf_pos = (pid % 10) + '0'; + pid /= 10; + } while (pid); + while (buf_pos < buf + 32 && out < out_end - 1) + *out++ = *buf_pos++; + s += 2; // skip "%p" + break; + } + default: + *out++ = *s++; + break; + } } + CHECK(out < out_end - 1); *out = '\0'; } @@ -67,7 +89,7 @@ class FlagHandlerInclude : public FlagHandlerBase { bool Parse(const char *value) final { if (internal_strchr(value, '%')) { char *buf = (char *)MmapOrDie(kMaxPathLength, "FlagHandlerInclude"); - SubstituteBinaryName(value, buf, kMaxPathLength); + SubstituteForFlagValue(value, buf, kMaxPathLength); bool res = parser_->ParseFile(buf, ignore_missing_); UnmapOrDie(buf, kMaxPathLength); return res; @@ -97,4 +119,10 @@ void RegisterCommonFlags(FlagParser *parser, CommonFlags *cf) { RegisterIncludeFlags(parser, cf); } +void InitializeCommonFlags(CommonFlags *cf) { + // need to record coverage to generate coverage report. + cf->coverage |= cf->html_cov_report; + SetVerbosity(cf->verbosity); +} + } // namespace __sanitizer diff --git a/libsanitizer/sanitizer_common/sanitizer_flags.h b/libsanitizer/sanitizer_common/sanitizer_flags.h index f1b872b79e2a..ff64af10ff5a 100644 --- a/libsanitizer/sanitizer_common/sanitizer_flags.h +++ b/libsanitizer/sanitizer_common/sanitizer_flags.h @@ -44,10 +44,17 @@ inline void OverrideCommonFlags(const CommonFlags &cf) { common_flags_dont_use.CopyFrom(cf); } +void SubstituteForFlagValue(const char *s, char *out, uptr out_size); + class FlagParser; void RegisterCommonFlags(FlagParser *parser, CommonFlags *cf = &common_flags_dont_use); void RegisterIncludeFlags(FlagParser *parser, CommonFlags *cf); + +// Should be called after parsing all flags. Sets up common flag values +// and perform initializations common to all sanitizers (e.g. setting +// verbosity). +void InitializeCommonFlags(CommonFlags *cf = &common_flags_dont_use); } // namespace __sanitizer #endif // SANITIZER_FLAGS_H diff --git a/libsanitizer/sanitizer_common/sanitizer_flags.inc b/libsanitizer/sanitizer_common/sanitizer_flags.inc index bca1c4b4d0ff..ccc0e416f4a6 100644 --- a/libsanitizer/sanitizer_common/sanitizer_flags.inc +++ b/libsanitizer/sanitizer_common/sanitizer_flags.inc @@ -54,7 +54,7 @@ COMMON_FLAG( "Mention name of executable when reporting error and " "append executable name to logs (as in \"log_path.exe_name.pid\").") COMMON_FLAG( - bool, log_to_syslog, SANITIZER_ANDROID, + bool, log_to_syslog, SANITIZER_ANDROID || SANITIZER_MAC, "Write all sanitizer output to syslog in addition to other means of " "logging.") COMMON_FLAG( @@ -73,10 +73,12 @@ COMMON_FLAG(bool, print_summary, true, "If false, disable printing error summaries in addition to error " "reports.") COMMON_FLAG(bool, check_printf, true, "Check printf arguments.") -COMMON_FLAG(bool, handle_segv, SANITIZER_NEEDS_SEGV, +COMMON_FLAG(bool, handle_segv, true, "If set, registers the tool's custom SIGSEGV/SIGBUS handler.") COMMON_FLAG(bool, handle_abort, false, "If set, registers the tool's custom SIGABRT handler.") +COMMON_FLAG(bool, handle_sigill, false, + "If set, registers the tool's custom SIGILL handler.") COMMON_FLAG(bool, handle_sigfpe, true, "If set, registers the tool's custom SIGFPE handler.") COMMON_FLAG(bool, allow_user_segv_handler, false, @@ -114,6 +116,10 @@ COMMON_FLAG(uptr, soft_rss_limit_mb, 0, " until the RSS goes below the soft limit." " This limit does not affect memory allocations other than" " malloc/new.") +COMMON_FLAG(bool, heap_profile, false, "Experimental heap profiler, asan-only") +COMMON_FLAG(bool, allocator_release_to_os, false, + "Experimental. If true, try to periodically release unused" + " memory to the OS.\n") COMMON_FLAG(bool, can_use_proc_maps_statm, true, "If false, do not attempt to read /proc/maps/statm." " Mostly useful for testing sanitizers.") @@ -146,10 +152,10 @@ COMMON_FLAG(bool, full_address_space, false, COMMON_FLAG(bool, print_suppressions, true, "Print matched suppressions at exit.") COMMON_FLAG( - bool, disable_coredump, (SANITIZER_WORDSIZE == 64), - "Disable core dumping. By default, disable_core=1 on 64-bit to avoid " - "dumping a 16T+ core file. Ignored on OSes that don't dump core by" - "default and for sanitizers that don't reserve lots of virtual memory.") + bool, disable_coredump, (SANITIZER_WORDSIZE == 64) && !SANITIZER_GO, + "Disable core dumping. By default, disable_coredump=1 on 64-bit to avoid" + " dumping a 16T+ core file. Ignored on OSes that don't dump core by" + " default and for sanitizers that don't reserve lots of virtual memory.") COMMON_FLAG(bool, use_madv_dontdump, true, "If set, instructs kernel to not store the (huge) shadow " "in core file.") @@ -158,6 +164,11 @@ COMMON_FLAG(bool, symbolize_inline_frames, true, COMMON_FLAG(bool, symbolize_vs_style, false, "Print file locations in Visual Studio style (e.g: " " file(10,42): ...") +COMMON_FLAG(int, dedup_token_length, 0, + "If positive, after printing a stack trace also print a short " + "string token based on this number of frames that will simplify " + "deduplication of the reports. " + "Example: 'DEDUP_TOKEN: foo-bar-main'. Default is 0.") COMMON_FLAG(const char *, stack_trace_format, "DEFAULT", "Format string used to render stack frames. " "See sanitizer_stacktrace_printer.h for the format description. " @@ -175,18 +186,42 @@ COMMON_FLAG(bool, intercept_strspn, true, COMMON_FLAG(bool, intercept_strpbrk, true, "If set, uses custom wrappers for strpbrk function " "to find more errors.") +COMMON_FLAG(bool, intercept_strlen, true, + "If set, uses custom wrappers for strlen and strnlen functions " + "to find more errors.") +COMMON_FLAG(bool, intercept_strchr, true, + "If set, uses custom wrappers for strchr, strchrnul, and strrchr " + "functions to find more errors.") COMMON_FLAG(bool, intercept_memcmp, true, "If set, uses custom wrappers for memcmp function " "to find more errors.") COMMON_FLAG(bool, strict_memcmp, true, "If true, assume that memcmp(p1, p2, n) always reads n bytes before " "comparing p1 and p2.") +COMMON_FLAG(bool, intercept_memmem, true, + "If set, uses a wrapper for memmem() to find more errors.") +COMMON_FLAG(bool, intercept_intrin, true, + "If set, uses custom wrappers for memset/memcpy/memmove " + "intrinsics to find more errors.") +COMMON_FLAG(bool, intercept_stat, true, + "If set, uses custom wrappers for *stat functions " + "to find more errors.") +COMMON_FLAG(bool, intercept_send, true, + "If set, uses custom wrappers for send* functions " + "to find more errors.") COMMON_FLAG(bool, decorate_proc_maps, false, "If set, decorate sanitizer " "mappings in /proc/self/maps with " "user-readable names") COMMON_FLAG(int, exitcode, 1, "Override the program exit status if the tool " "found an error") COMMON_FLAG( - bool, abort_on_error, SANITIZER_MAC, + bool, abort_on_error, SANITIZER_ANDROID || SANITIZER_MAC, "If set, the tool calls abort() instead of _exit() after printing the " "error report.") +COMMON_FLAG(bool, suppress_equal_pcs, true, + "Deduplicate multiple reports for single source location in " + "halt_on_error=false mode (asan only).") +COMMON_FLAG(bool, print_cmdline, false, "Print command line on crash " + "(asan only).") +COMMON_FLAG(bool, html_cov_report, false, "Generate html coverage report.") +COMMON_FLAG(const char *, sancov_path, "sancov", "Sancov tool location.") diff --git a/libsanitizer/sanitizer_common/sanitizer_interface_internal.h b/libsanitizer/sanitizer_common/sanitizer_interface_internal.h index 0547f9927cb3..34e80c3b50f8 100644 --- a/libsanitizer/sanitizer_common/sanitizer_interface_internal.h +++ b/libsanitizer/sanitizer_common/sanitizer_interface_internal.h @@ -23,6 +23,10 @@ extern "C" { // The special values are "stdout" and "stderr". SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_set_report_path(const char *path); + // Tell the tools to write their reports to the provided file descriptor + // (casted to void *). + SANITIZER_INTERFACE_ATTRIBUTE + void __sanitizer_set_report_fd(void *fd); typedef struct { int coverage_sandboxed; diff --git a/libsanitizer/sanitizer_common/sanitizer_internal_defs.h b/libsanitizer/sanitizer_common/sanitizer_internal_defs.h index d76ed7570a7e..b9c906669dee 100644 --- a/libsanitizer/sanitizer_common/sanitizer_internal_defs.h +++ b/libsanitizer/sanitizer_common/sanitizer_internal_defs.h @@ -22,7 +22,7 @@ # define SANITIZER_INTERFACE_ATTRIBUTE __declspec(dllexport) // FIXME find out what we need on Windows, if anything. # define SANITIZER_WEAK_ATTRIBUTE -#elif defined(SANITIZER_GO) +#elif SANITIZER_GO # define SANITIZER_INTERFACE_ATTRIBUTE # define SANITIZER_WEAK_ATTRIBUTE #else @@ -30,7 +30,7 @@ # define SANITIZER_WEAK_ATTRIBUTE __attribute__((weak)) #endif -#if (SANITIZER_LINUX || SANITIZER_WINDOWS) && !defined(SANITIZER_GO) +#if (SANITIZER_LINUX || SANITIZER_WINDOWS) && !SANITIZER_GO # define SANITIZER_SUPPORTS_WEAK_HOOKS 1 #else # define SANITIZER_SUPPORTS_WEAK_HOOKS 0 @@ -87,6 +87,7 @@ typedef unsigned error_t; typedef int fd_t; typedef int error_t; #endif +typedef int pid_t; // WARNING: OFF_T may be different from OS type off_t, depending on the value of // _FILE_OFFSET_BITS. This definition of OFF_T matches the ABI of system calls @@ -103,19 +104,25 @@ typedef u64 OFF64_T; #if (SANITIZER_WORDSIZE == 64) || SANITIZER_MAC typedef uptr operator_new_size_type; #else +# if defined(__s390__) && !defined(__s390x__) +// Special case: 31-bit s390 has unsigned long as size_t. +typedef unsigned long operator_new_size_type; +# else typedef u32 operator_new_size_type; +# endif #endif -} // namespace __sanitizer -using namespace __sanitizer; // NOLINT // ----------- ATTENTION ------------- // This header should NOT include any other headers to avoid portability issues. // Common defs. #define INLINE inline #define INTERFACE_ATTRIBUTE SANITIZER_INTERFACE_ATTRIBUTE -#define WEAK SANITIZER_WEAK_ATTRIBUTE +#define SANITIZER_WEAK_DEFAULT_IMPL \ + extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE NOINLINE +#define SANITIZER_WEAK_CXX_DEFAULT_IMPL \ + extern "C++" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE NOINLINE // Platform-specific defs. #if defined(_MSC_VER) @@ -129,7 +136,7 @@ using namespace __sanitizer; // NOLINT # define THREADLOCAL __declspec(thread) # define LIKELY(x) (x) # define UNLIKELY(x) (x) -# define PREFETCH(x) /* _mm_prefetch(x, _MM_HINT_NTA) */ +# define PREFETCH(x) /* _mm_prefetch(x, _MM_HINT_NTA) */ (void)0 #else // _MSC_VER # define ALWAYS_INLINE inline __attribute__((always_inline)) # define ALIAS(x) __attribute__((alias(x))) @@ -173,7 +180,9 @@ typedef ALIGNED(1) s32 us32; typedef ALIGNED(1) s64 us64; #if SANITIZER_WINDOWS +} // namespace __sanitizer typedef unsigned long DWORD; // NOLINT +namespace __sanitizer { typedef DWORD thread_return_t; # define THREAD_CALLING_CONV __stdcall #else // _WIN32 @@ -183,14 +192,12 @@ typedef void* thread_return_t; typedef thread_return_t (THREAD_CALLING_CONV *thread_callback_t)(void* arg); // NOTE: Functions below must be defined in each run-time. -namespace __sanitizer { void NORETURN Die(); // FIXME: No, this shouldn't be in the sanitizer interface. SANITIZER_INTERFACE_ATTRIBUTE void NORETURN CheckFailed(const char *file, int line, const char *cond, u64 v1, u64 v2); -} // namespace __sanitizer // Check macro #define RAW_CHECK_MSG(expr, msg) do { \ @@ -282,14 +289,23 @@ enum LinkerInitialized { LINKER_INITIALIZED = 0 }; #if !defined(_MSC_VER) || defined(__clang__) # define GET_CALLER_PC() (uptr)__builtin_return_address(0) # define GET_CURRENT_FRAME() (uptr)__builtin_frame_address(0) +inline void Trap() { + __builtin_trap(); +} #else extern "C" void* _ReturnAddress(void); +extern "C" void* _AddressOfReturnAddress(void); # pragma intrinsic(_ReturnAddress) +# pragma intrinsic(_AddressOfReturnAddress) # define GET_CALLER_PC() (uptr)_ReturnAddress() // CaptureStackBackTrace doesn't need to know BP on Windows. -// FIXME: This macro is still used when printing error reports though it's not -// clear if the BP value is needed in the ASan reports on Windows. -# define GET_CURRENT_FRAME() (uptr)0xDEADBEEF +# define GET_CURRENT_FRAME() (((uptr)_AddressOfReturnAddress()) + sizeof(uptr)) + +extern "C" void __ud2(void); +# pragma intrinsic(__ud2) +inline void Trap() { + __ud2(); +} #endif #define HANDLE_EINTR(res, f) \ @@ -308,4 +324,19 @@ extern "C" void* _ReturnAddress(void); (void)enable_fp; \ } while (0) +} // namespace __sanitizer + +namespace __asan { using namespace __sanitizer; } // NOLINT +namespace __dsan { using namespace __sanitizer; } // NOLINT +namespace __dfsan { using namespace __sanitizer; } // NOLINT +namespace __esan { using namespace __sanitizer; } // NOLINT +namespace __lsan { using namespace __sanitizer; } // NOLINT +namespace __msan { using namespace __sanitizer; } // NOLINT +namespace __tsan { using namespace __sanitizer; } // NOLINT +namespace __scudo { using namespace __sanitizer; } // NOLINT +namespace __ubsan { using namespace __sanitizer; } // NOLINT +namespace __xray { using namespace __sanitizer; } // NOLINT +namespace __interception { using namespace __sanitizer; } // NOLINT + + #endif // SANITIZER_DEFS_H diff --git a/libsanitizer/sanitizer_common/sanitizer_libc.cc b/libsanitizer/sanitizer_common/sanitizer_libc.cc index 05fdca622e43..82d37675e479 100644 --- a/libsanitizer/sanitizer_common/sanitizer_libc.cc +++ b/libsanitizer/sanitizer_common/sanitizer_libc.cc @@ -72,7 +72,7 @@ void *internal_memmove(void *dest, const void *src, uptr n) { // Semi-fast bzero for 16-aligned data. Still far from peak performance. void internal_bzero_aligned16(void *s, uptr n) { - struct S16 { u64 a, b; } ALIGNED(16); + struct ALIGNED(16) S16 { u64 a, b; }; CHECK_EQ((reinterpret_cast<uptr>(s) | n) & 15, 0); for (S16 *p = reinterpret_cast<S16*>(s), *end = p + n / 16; p < end; p++) { p->a = p->b = 0; @@ -173,6 +173,19 @@ uptr internal_strlen(const char *s) { return i; } +uptr internal_strlcat(char *dst, const char *src, uptr maxlen) { + const uptr srclen = internal_strlen(src); + const uptr dstlen = internal_strnlen(dst, maxlen); + if (dstlen == maxlen) return maxlen + srclen; + if (srclen < maxlen - dstlen) { + internal_memmove(dst + dstlen, src, srclen + 1); + } else { + internal_memmove(dst + dstlen, src, maxlen - dstlen - 1); + dst[maxlen - 1] = '\0'; + } + return dstlen + srclen; +} + char *internal_strncat(char *dst, const char *src, uptr n) { uptr len = internal_strlen(dst); uptr i; @@ -182,6 +195,17 @@ char *internal_strncat(char *dst, const char *src, uptr n) { return dst; } +uptr internal_strlcpy(char *dst, const char *src, uptr maxlen) { + const uptr srclen = internal_strlen(src); + if (srclen < maxlen) { + internal_memmove(dst, src, srclen + 1); + } else if (maxlen != 0) { + internal_memmove(dst, src, maxlen - 1); + dst[maxlen - 1] = '\0'; + } + return srclen; +} + char *internal_strncpy(char *dst, const char *src, uptr n) { uptr i; for (i = 0; i < n && src[i]; i++) @@ -208,6 +232,12 @@ char *internal_strstr(const char *haystack, const char *needle) { return nullptr; } +uptr internal_wcslen(const wchar_t *s) { + uptr i = 0; + while (s[i]) i++; + return i; +} + s64 internal_simple_strtoll(const char *nptr, char **endptr, int base) { CHECK_EQ(base, 10); while (IsSpace(*nptr)) nptr++; diff --git a/libsanitizer/sanitizer_common/sanitizer_libc.h b/libsanitizer/sanitizer_common/sanitizer_libc.h index 1b3f8edfadbe..d26ec8704c08 100644 --- a/libsanitizer/sanitizer_common/sanitizer_libc.h +++ b/libsanitizer/sanitizer_common/sanitizer_libc.h @@ -41,12 +41,15 @@ uptr internal_strcspn(const char *s, const char *reject); char *internal_strdup(const char *s); char *internal_strndup(const char *s, uptr n); uptr internal_strlen(const char *s); +uptr internal_strlcat(char *dst, const char *src, uptr maxlen); char *internal_strncat(char *dst, const char *src, uptr n); int internal_strncmp(const char *s1, const char *s2, uptr n); +uptr internal_strlcpy(char *dst, const char *src, uptr maxlen); char *internal_strncpy(char *dst, const char *src, uptr n); uptr internal_strnlen(const char *s, uptr maxlen); char *internal_strrchr(const char *s, int c); // This is O(N^2), but we are not using it in hot places. +uptr internal_wcslen(const wchar_t *s); char *internal_strstr(const char *haystack, const char *needle); // Works only for base=10 and doesn't set errno. s64 internal_simple_strtoll(const char *nptr, char **endptr, int base); @@ -57,15 +60,18 @@ int internal_snprintf(char *buffer, uptr length, const char *format, ...); bool mem_is_zero(const char *mem, uptr size); // I/O -const fd_t kInvalidFd = (fd_t)-1; -const fd_t kStdinFd = 0; -const fd_t kStdoutFd = (fd_t)1; -const fd_t kStderrFd = (fd_t)2; +// Define these as macros so we can use them in linker initialized global +// structs without dynamic initialization. +#define kInvalidFd ((fd_t)-1) +#define kStdinFd ((fd_t)0) +#define kStdoutFd ((fd_t)1) +#define kStderrFd ((fd_t)2) uptr internal_ftruncate(fd_t fd, uptr size); // OS void NORETURN internal__exit(int exitcode); +unsigned int internal_sleep(unsigned int seconds); uptr internal_getpid(); uptr internal_getppid(); diff --git a/libsanitizer/sanitizer_common/sanitizer_linux.cc b/libsanitizer/sanitizer_common/sanitizer_linux.cc index 2cefa20a5f08..806fcd5e2847 100644 --- a/libsanitizer/sanitizer_common/sanitizer_linux.cc +++ b/libsanitizer/sanitizer_common/sanitizer_linux.cc @@ -58,7 +58,10 @@ #include <unistd.h> #if SANITIZER_FREEBSD +#include <sys/exec.h> #include <sys/sysctl.h> +#include <vm/vm_param.h> +#include <vm/pmap.h> #include <machine/atomic.h> extern "C" { // <sys/umtx.h> must be included after <errno.h> and <sys/types.h> on @@ -87,12 +90,19 @@ const int FUTEX_WAKE = 1; // Are we using 32-bit or 64-bit Linux syscalls? // x32 (which defines __x86_64__) has SANITIZER_WORDSIZE == 32 // but it still needs to use 64-bit syscalls. -#if SANITIZER_LINUX && (defined(__x86_64__) || SANITIZER_WORDSIZE == 64) +#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__powerpc64__) || \ + SANITIZER_WORDSIZE == 64) # define SANITIZER_LINUX_USES_64BIT_SYSCALLS 1 #else # define SANITIZER_LINUX_USES_64BIT_SYSCALLS 0 #endif +#if defined(__x86_64__) || SANITIZER_MIPS64 +extern "C" { +extern void internal_sigreturn(); +} +#endif + namespace __sanitizer { #if SANITIZER_LINUX && defined(__x86_64__) @@ -104,6 +114,7 @@ namespace __sanitizer { #endif // --------------- sanitizer_libc.h +#if !SANITIZER_S390 uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd, OFF_T offset) { #if SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS @@ -116,6 +127,7 @@ uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd, offset / 4096); #endif } +#endif // !SANITIZER_S390 uptr internal_munmap(void *addr, uptr length) { return internal_syscall(SYSCALL(munmap), (uptr)addr, length); @@ -238,7 +250,15 @@ uptr internal_lstat(const char *path, void *buf) { return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf, AT_SYMLINK_NOFOLLOW); #elif SANITIZER_LINUX_USES_64BIT_SYSCALLS +# if SANITIZER_MIPS64 + // For mips64, lstat syscall fills buffer in the format of kernel_stat + struct kernel_stat kbuf; + int res = internal_syscall(SYSCALL(lstat), path, &kbuf); + kernel_stat_to_stat(&kbuf, (struct stat *)buf); + return res; +# else return internal_syscall(SYSCALL(lstat), (uptr)path, (uptr)buf); +# endif #else struct stat64 buf64; int res = internal_syscall(SYSCALL(lstat64), path, &buf64); @@ -249,7 +269,15 @@ uptr internal_lstat(const char *path, void *buf) { uptr internal_fstat(fd_t fd, void *buf) { #if SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS +# if SANITIZER_MIPS64 + // For mips64, fstat syscall fills buffer in the format of kernel_stat + struct kernel_stat kbuf; + int res = internal_syscall(SYSCALL(fstat), fd, &kbuf); + kernel_stat_to_stat(&kbuf, (struct stat *)buf); + return res; +# else return internal_syscall(SYSCALL(fstat), fd, (uptr)buf); +# endif #else struct stat64 buf64; int res = internal_syscall(SYSCALL(fstat64), fd, &buf64); @@ -312,6 +340,15 @@ void internal__exit(int exitcode) { Die(); // Unreachable. } +unsigned int internal_sleep(unsigned int seconds) { + struct timespec ts; + ts.tv_sec = 1; + ts.tv_nsec = 0; + int res = internal_syscall(SYSCALL(nanosleep), &ts, &ts); + if (res) return ts.tv_sec; + return 0; +} + uptr internal_execve(const char *filename, char *const argv[], char *const envp[]) { return internal_syscall(SYSCALL(execve), (uptr)filename, (uptr)argv, @@ -392,11 +429,13 @@ const char *GetEnv(const char *name) { #endif } +#if !SANITIZER_FREEBSD extern "C" { SANITIZER_WEAK_ATTRIBUTE extern void *__libc_stack_end; } +#endif -#if !SANITIZER_GO +#if !SANITIZER_GO && !SANITIZER_FREEBSD static void ReadNullSepFileToArray(const char *path, char ***arr, int arr_size) { char *buff; @@ -421,7 +460,8 @@ static void ReadNullSepFileToArray(const char *path, char ***arr, } #endif -static void GetArgsAndEnv(char*** argv, char*** envp) { +static void GetArgsAndEnv(char ***argv, char ***envp) { +#if !SANITIZER_FREEBSD #if !SANITIZER_GO if (&__libc_stack_end) { #endif @@ -436,6 +476,25 @@ static void GetArgsAndEnv(char*** argv, char*** envp) { ReadNullSepFileToArray("/proc/self/environ", envp, kMaxEnvp); } #endif +#else + // On FreeBSD, retrieving the argument and environment arrays is done via the + // kern.ps_strings sysctl, which returns a pointer to a structure containing + // this information. See also <sys/exec.h>. + ps_strings *pss; + size_t sz = sizeof(pss); + if (sysctlbyname("kern.ps_strings", &pss, &sz, NULL, 0) == -1) { + Printf("sysctl kern.ps_strings failed\n"); + Die(); + } + *argv = pss->ps_argvstr; + *envp = pss->ps_envstr; +#endif +} + +char **GetArgv() { + char **argv, **envp; + GetArgsAndEnv(&argv, &envp); + return argv; } void ReExec() { @@ -561,7 +620,8 @@ int internal_fork() { #if SANITIZER_LINUX #define SA_RESTORER 0x04000000 -// Doesn't set sa_restorer, use with caution (see below). +// Doesn't set sa_restorer if the caller did not set it, so use with caution +//(see below). int internal_sigaction_norestorer(int signum, const void *act, void *oldact) { __sanitizer_kernel_sigaction_t k_act, k_oldact; internal_memset(&k_act, 0, sizeof(__sanitizer_kernel_sigaction_t)); @@ -583,7 +643,9 @@ int internal_sigaction_norestorer(int signum, const void *act, void *oldact) { // rt_sigaction, so we need to do the same (we'll need to reimplement the // restorers; for x86_64 the restorer address can be obtained from // oldact->sa_restorer upon a call to sigaction(xxx, NULL, oldact). +#if !SANITIZER_ANDROID || !SANITIZER_MIPS32 k_act.sa_restorer = u_act->sa_restorer; +#endif } uptr result = internal_syscall(SYSCALL(rt_sigaction), (uptr)signum, @@ -597,10 +659,31 @@ int internal_sigaction_norestorer(int signum, const void *act, void *oldact) { internal_memcpy(&u_oldact->sa_mask, &k_oldact.sa_mask, sizeof(__sanitizer_kernel_sigset_t)); u_oldact->sa_flags = k_oldact.sa_flags; +#if !SANITIZER_ANDROID || !SANITIZER_MIPS32 u_oldact->sa_restorer = k_oldact.sa_restorer; +#endif } return result; } + +// Invokes sigaction via a raw syscall with a restorer, but does not support +// all platforms yet. +// We disable for Go simply because we have not yet added to buildgo.sh. +#if (defined(__x86_64__) || SANITIZER_MIPS64) && !SANITIZER_GO +int internal_sigaction_syscall(int signum, const void *act, void *oldact) { + if (act == nullptr) + return internal_sigaction_norestorer(signum, act, oldact); + __sanitizer_sigaction u_adjust; + internal_memcpy(&u_adjust, act, sizeof(u_adjust)); +#if !SANITIZER_ANDROID || !SANITIZER_MIPS32 + if (u_adjust.sa_restorer == nullptr) { + u_adjust.sa_restorer = internal_sigreturn; + } +#endif + return internal_sigaction_norestorer(signum, (const void *)&u_adjust, + oldact); +} +#endif // defined(__x86_64__) && !SANITIZER_GO #endif // SANITIZER_LINUX uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set, @@ -620,6 +703,10 @@ void internal_sigfillset(__sanitizer_sigset_t *set) { internal_memset(set, 0xff, sizeof(*set)); } +void internal_sigemptyset(__sanitizer_sigset_t *set) { + internal_memset(set, 0, sizeof(*set)); +} + #if SANITIZER_LINUX void internal_sigdelset(__sanitizer_sigset_t *set, int signum) { signum -= 1; @@ -630,6 +717,16 @@ void internal_sigdelset(__sanitizer_sigset_t *set, int signum) { const uptr bit = signum % (sizeof(k_set->sig[0]) * 8); k_set->sig[idx] &= ~(1 << bit); } + +bool internal_sigismember(__sanitizer_sigset_t *set, int signum) { + signum -= 1; + CHECK_GE(signum, 0); + CHECK_LT(signum, sizeof(*set) * 8); + __sanitizer_kernel_sigset_t *k_set = (__sanitizer_kernel_sigset_t *)set; + const uptr idx = signum / (sizeof(k_set->sig[0]) * 8); + const uptr bit = signum % (sizeof(k_set->sig[0]) * 8); + return k_set->sig[idx] & (1 << bit); +} #endif // SANITIZER_LINUX // ThreadLister implementation. @@ -701,7 +798,10 @@ bool ThreadLister::GetDirectoryEntries() { } uptr GetPageSize() { -#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__i386__)) +// Android post-M sysconf(_SC_PAGESIZE) crashes if called from .preinit_array. +#if SANITIZER_ANDROID + return 4096; +#elif SANITIZER_LINUX && (defined(__x86_64__) || defined(__i386__)) return EXEC_PAGESIZE; #else return sysconf(_SC_PAGESIZE); // EXEC_PAGESIZE may not be trustworthy. @@ -908,8 +1008,18 @@ uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, "bnez $2,1f;\n" /* Call "fn(arg)". */ +#if SANITIZER_WORDSIZE == 32 +#ifdef __BIG_ENDIAN__ + "lw $25,4($29);\n" + "lw $4,12($29);\n" +#else + "lw $25,0($29);\n" + "lw $4,8($29);\n" +#endif +#else "ld $25,0($29);\n" "ld $4,8($29);\n" +#endif "jal $25;\n" /* Call _exit($v0). */ @@ -981,18 +1091,91 @@ uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, : "x30", "memory"); return res; } -#endif // defined(__x86_64__) && SANITIZER_LINUX +#elif defined(__powerpc64__) +uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, + int *parent_tidptr, void *newtls, int *child_tidptr) { + long long res; +/* Stack frame offsets. */ +#if _CALL_ELF != 2 +#define FRAME_MIN_SIZE 112 +#define FRAME_TOC_SAVE 40 +#else +#define FRAME_MIN_SIZE 32 +#define FRAME_TOC_SAVE 24 +#endif + if (!fn || !child_stack) + return -EINVAL; + CHECK_EQ(0, (uptr)child_stack % 16); + child_stack = (char *)child_stack - 2 * sizeof(unsigned long long); + ((unsigned long long *)child_stack)[0] = (uptr)fn; + ((unsigned long long *)child_stack)[1] = (uptr)arg; -#if SANITIZER_ANDROID -#define PROP_VALUE_MAX 92 -extern "C" SANITIZER_WEAK_ATTRIBUTE int __system_property_get(const char *name, - char *value); -void GetExtraActivationFlags(char *buf, uptr size) { - CHECK(size > PROP_VALUE_MAX); - CHECK(&__system_property_get); - __system_property_get("asan.options", buf); + register int (*__fn)(void *) __asm__("r3") = fn; + register void *__cstack __asm__("r4") = child_stack; + register int __flags __asm__("r5") = flags; + register void * __arg __asm__("r6") = arg; + register int * __ptidptr __asm__("r7") = parent_tidptr; + register void * __newtls __asm__("r8") = newtls; + register int * __ctidptr __asm__("r9") = child_tidptr; + + __asm__ __volatile__( + /* fn, arg, child_stack are saved acrVoss the syscall */ + "mr 28, %5\n\t" + "mr 29, %6\n\t" + "mr 27, %8\n\t" + + /* syscall + r3 == flags + r4 == child_stack + r5 == parent_tidptr + r6 == newtls + r7 == child_tidptr */ + "mr 3, %7\n\t" + "mr 5, %9\n\t" + "mr 6, %10\n\t" + "mr 7, %11\n\t" + "li 0, %3\n\t" + "sc\n\t" + + /* Test if syscall was successful */ + "cmpdi cr1, 3, 0\n\t" + "crandc cr1*4+eq, cr1*4+eq, cr0*4+so\n\t" + "bne- cr1, 1f\n\t" + + /* Do the function call */ + "std 2, %13(1)\n\t" +#if _CALL_ELF != 2 + "ld 0, 0(28)\n\t" + "ld 2, 8(28)\n\t" + "mtctr 0\n\t" +#else + "mr 12, 28\n\t" + "mtctr 12\n\t" +#endif + "mr 3, 27\n\t" + "bctrl\n\t" + "ld 2, %13(1)\n\t" + + /* Call _exit(r3) */ + "li 0, %4\n\t" + "sc\n\t" + + /* Return to parent */ + "1:\n\t" + "mr %0, 3\n\t" + : "=r" (res) + : "0" (-1), "i" (EINVAL), + "i" (__NR_clone), "i" (__NR_exit), + "r" (__fn), "r" (__cstack), "r" (__flags), + "r" (__arg), "r" (__ptidptr), "r" (__newtls), + "r" (__ctidptr), "i" (FRAME_MIN_SIZE), "i" (FRAME_TOC_SAVE) + : "cr0", "cr1", "memory", "ctr", + "r0", "r29", "r27", "r28"); + return res; } +#endif // defined(__x86_64__) && SANITIZER_LINUX +#if SANITIZER_ANDROID #if __ANDROID_API__ < 21 extern "C" __attribute__((weak)) int dl_iterate_phdr( int (*)(struct dl_phdr_info *, size_t, void *), void *); @@ -1035,15 +1218,17 @@ AndroidApiLevel AndroidGetApiLevel() { #endif -bool IsDeadlySignal(int signum) { +bool IsHandledDeadlySignal(int signum) { if (common_flags()->handle_abort && signum == SIGABRT) return true; + if (common_flags()->handle_sigill && signum == SIGILL) + return true; if (common_flags()->handle_sigfpe && signum == SIGFPE) return true; return (signum == SIGSEGV || signum == SIGBUS) && common_flags()->handle_segv; } -#ifndef SANITIZER_GO +#if !SANITIZER_GO void *internal_start_thread(void(*func)(void *arg), void *arg) { // Start the thread with signals blocked, otherwise it can steal user signals. __sanitizer_sigset_t set, old; @@ -1069,6 +1254,54 @@ void *internal_start_thread(void (*func)(void *), void *arg) { return 0; } void internal_join_thread(void *th) {} #endif +#if defined(__aarch64__) +// Android headers in the older NDK releases miss this definition. +struct __sanitizer_esr_context { + struct _aarch64_ctx head; + uint64_t esr; +}; + +static bool Aarch64GetESR(ucontext_t *ucontext, u64 *esr) { + static const u32 kEsrMagic = 0x45535201; + u8 *aux = ucontext->uc_mcontext.__reserved; + while (true) { + _aarch64_ctx *ctx = (_aarch64_ctx *)aux; + if (ctx->size == 0) break; + if (ctx->magic == kEsrMagic) { + *esr = ((__sanitizer_esr_context *)ctx)->esr; + return true; + } + aux += ctx->size; + } + return false; +} +#endif + +SignalContext::WriteFlag SignalContext::GetWriteFlag(void *context) { + ucontext_t *ucontext = (ucontext_t *)context; +#if defined(__x86_64__) || defined(__i386__) + static const uptr PF_WRITE = 1U << 1; +#if SANITIZER_FREEBSD + uptr err = ucontext->uc_mcontext.mc_err; +#else + uptr err = ucontext->uc_mcontext.gregs[REG_ERR]; +#endif + return err & PF_WRITE ? WRITE : READ; +#elif defined(__arm__) + static const uptr FSR_WRITE = 1U << 11; + uptr fsr = ucontext->uc_mcontext.error_code; + return fsr & FSR_WRITE ? WRITE : READ; +#elif defined(__aarch64__) + static const u64 ESR_ELx_WNR = 1U << 6; + u64 esr; + if (!Aarch64GetESR(ucontext, &esr)) return UNKNOWN; + return esr & ESR_ELx_WNR ? WRITE : READ; +#else + (void)ucontext; + return UNKNOWN; // FIXME: Implement. +#endif +} + void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { #if defined(__arm__) ucontext_t *ucontext = (ucontext_t*)context; @@ -1136,11 +1369,29 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { *pc = ucontext->uc_mcontext.pc; *bp = ucontext->uc_mcontext.gregs[30]; *sp = ucontext->uc_mcontext.gregs[29]; +#elif defined(__s390__) + ucontext_t *ucontext = (ucontext_t*)context; +# if defined(__s390x__) + *pc = ucontext->uc_mcontext.psw.addr; +# else + *pc = ucontext->uc_mcontext.psw.addr & 0x7fffffff; +# endif + *bp = ucontext->uc_mcontext.gregs[11]; + *sp = ucontext->uc_mcontext.gregs[15]; #else # error "Unsupported arch" #endif } +void MaybeReexec() { + // No need to re-exec on Linux. +} + +uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding) { + UNREACHABLE("FindAvailableMemoryRange is not available"); + return 0; +} + } // namespace __sanitizer #endif // SANITIZER_FREEBSD || SANITIZER_LINUX diff --git a/libsanitizer/sanitizer_common/sanitizer_linux.h b/libsanitizer/sanitizer_common/sanitizer_linux.h index 44977020bce4..895bfc18195c 100644 --- a/libsanitizer/sanitizer_common/sanitizer_linux.h +++ b/libsanitizer/sanitizer_common/sanitizer_linux.h @@ -32,7 +32,6 @@ uptr internal_sigaltstack(const struct sigaltstack* ss, struct sigaltstack* oss); uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset); -void internal_sigfillset(__sanitizer_sigset_t *set); // Linux-only syscalls. #if SANITIZER_LINUX @@ -41,8 +40,13 @@ uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5); // (like the process-wide error reporting SEGV handler) must use // internal_sigaction instead. int internal_sigaction_norestorer(int signum, const void *act, void *oldact); +#if (defined(__x86_64__) || SANITIZER_MIPS64) && !SANITIZER_GO +// Uses a raw system call to avoid interceptors. +int internal_sigaction_syscall(int signum, const void *act, void *oldact); +#endif void internal_sigdelset(__sanitizer_sigset_t *set, int signum); -#if defined(__x86_64__) || defined(__mips__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__mips__) || defined(__aarch64__) \ + || defined(__powerpc64__) || defined(__s390__) uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, int *parent_tidptr, void *newtls, int *child_tidptr); #endif diff --git a/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cc index ff69664e7b99..63e70660cf35 100644 --- a/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cc +++ b/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cc @@ -32,6 +32,7 @@ #include <pthread.h> #include <signal.h> #include <sys/resource.h> +#include <syslog.h> #if SANITIZER_FREEBSD #include <pthread_np.h> @@ -49,8 +50,6 @@ #if SANITIZER_ANDROID && __ANDROID_API__ < 21 #include <android/log.h> -#else -#include <syslog.h> #endif #if !SANITIZER_ANDROID @@ -156,7 +155,6 @@ bool SanitizerGetThreadName(char *name, int max_len) { #if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO static uptr g_tls_size; -#endif #ifdef __i386__ # define DL_INTERNAL_FUNCTION __attribute__((regparm(3), stdcall)) @@ -164,26 +162,7 @@ static uptr g_tls_size; # define DL_INTERNAL_FUNCTION #endif -#if defined(__mips__) || defined(__powerpc64__) -// TlsPreTcbSize includes size of struct pthread_descr and size of tcb -// head structure. It lies before the static tls blocks. -static uptr TlsPreTcbSize() { -# if defined(__mips__) - const uptr kTcbHead = 16; // sizeof (tcbhead_t) -# elif defined(__powerpc64__) - const uptr kTcbHead = 88; // sizeof (tcbhead_t) -# endif - const uptr kTlsAlign = 16; - const uptr kTlsPreTcbSize = - (ThreadDescriptorSize() + kTcbHead + kTlsAlign - 1) & ~(kTlsAlign - 1); - InitTlsSize(); - g_tls_size = (g_tls_size + kTlsPreTcbSize + kTlsAlign -1) & ~(kTlsAlign - 1); - return kTlsPreTcbSize; -} -#endif - void InitTlsSize() { -#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO // all current supported platforms have 16 bytes stack alignment const size_t kStackAlign = 16; typedef void (*get_tls_func)(size_t*, size_t*) DL_INTERNAL_FUNCTION; @@ -199,11 +178,13 @@ void InitTlsSize() { if (tls_align < kStackAlign) tls_align = kStackAlign; g_tls_size = RoundUpTo(tls_size, tls_align); -#endif // !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO } +#else +void InitTlsSize() { } +#endif // !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO #if (defined(__x86_64__) || defined(__i386__) || defined(__mips__) \ - || defined(__aarch64__) || defined(__powerpc64__)) \ + || defined(__aarch64__) || defined(__powerpc64__) || defined(__s390__)) \ && SANITIZER_LINUX && !SANITIZER_ANDROID // sizeof(struct pthread) from glibc. static atomic_uintptr_t kThreadDescriptorSize; @@ -220,6 +201,11 @@ uptr ThreadDescriptorSize() { char *end; int minor = internal_simple_strtoll(buf + 8, &end, 10); if (end != buf + 8 && (*end == '\0' || *end == '.')) { + int patch = 0; + if (*end == '.') + // strtoll will return 0 if no valid conversion could be performed + patch = internal_simple_strtoll(end + 1, nullptr, 10); + /* sizeof(struct pthread) values from various glibc versions. */ if (SANITIZER_X32) val = 1728; // Assume only one particular version for x32. @@ -233,9 +219,9 @@ uptr ThreadDescriptorSize() { val = FIRST_32_SECOND_64(1136, 1712); else if (minor == 10) val = FIRST_32_SECOND_64(1168, 1776); - else if (minor <= 12) + else if (minor == 11 || (minor == 12 && patch == 1)) val = FIRST_32_SECOND_64(1168, 2288); - else if (minor == 13) + else if (minor <= 13) val = FIRST_32_SECOND_64(1168, 2304); else val = FIRST_32_SECOND_64(1216, 2304); @@ -260,6 +246,9 @@ uptr ThreadDescriptorSize() { val = 1776; // from glibc.ppc64le 2.20-8.fc21 atomic_store(&kThreadDescriptorSize, val, memory_order_relaxed); return val; +#elif defined(__s390__) + val = FIRST_32_SECOND_64(1152, 1776); // valid for glibc 2.22 + atomic_store(&kThreadDescriptorSize, val, memory_order_relaxed); #endif return 0; } @@ -271,6 +260,24 @@ uptr ThreadSelfOffset() { return kThreadSelfOffset; } +#if defined(__mips__) || defined(__powerpc64__) +// TlsPreTcbSize includes size of struct pthread_descr and size of tcb +// head structure. It lies before the static tls blocks. +static uptr TlsPreTcbSize() { +# if defined(__mips__) + const uptr kTcbHead = 16; // sizeof (tcbhead_t) +# elif defined(__powerpc64__) + const uptr kTcbHead = 88; // sizeof (tcbhead_t) +# endif + const uptr kTlsAlign = 16; + const uptr kTlsPreTcbSize = + (ThreadDescriptorSize() + kTcbHead + kTlsAlign - 1) & ~(kTlsAlign - 1); + InitTlsSize(); + g_tls_size = (g_tls_size + kTlsPreTcbSize + kTlsAlign -1) & ~(kTlsAlign - 1); + return kTlsPreTcbSize; +} +#endif + uptr ThreadSelf() { uptr descr_addr; # if defined(__i386__) @@ -290,6 +297,9 @@ uptr ThreadSelf() { .set pop" : "=r" (thread_pointer)); descr_addr = thread_pointer - kTlsTcbOffset - TlsPreTcbSize(); # elif defined(__aarch64__) + descr_addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) - + ThreadDescriptorSize(); +# elif defined(__s390__) descr_addr = reinterpret_cast<uptr>(__builtin_thread_pointer()); # elif defined(__powerpc64__) // PPC64LE uses TLS variant I. The thread pointer (in GPR 13) @@ -330,7 +340,7 @@ uptr ThreadSelf() { #if !SANITIZER_GO static void GetTls(uptr *addr, uptr *size) { #if SANITIZER_LINUX && !SANITIZER_ANDROID -# if defined(__x86_64__) || defined(__i386__) +# if defined(__x86_64__) || defined(__i386__) || defined(__s390__) *addr = ThreadSelf(); *size = GetTlsSize(); *addr -= *size; @@ -410,17 +420,12 @@ typedef ElfW(Phdr) Elf_Phdr; # endif struct DlIteratePhdrData { - LoadedModule *modules; - uptr current_n; + InternalMmapVector<LoadedModule> *modules; bool first; - uptr max_n; - string_predicate_t filter; }; static int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *arg) { DlIteratePhdrData *data = (DlIteratePhdrData*)arg; - if (data->current_n == data->max_n) - return 0; InternalScopedString module_name(kMaxPathLength); if (data->first) { data->first = false; @@ -431,20 +436,18 @@ static int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *arg) { } if (module_name[0] == '\0') return 0; - if (data->filter && !data->filter(module_name.data())) - return 0; - LoadedModule *cur_module = &data->modules[data->current_n]; - cur_module->set(module_name.data(), info->dlpi_addr); - data->current_n++; + LoadedModule cur_module; + cur_module.set(module_name.data(), info->dlpi_addr); for (int i = 0; i < info->dlpi_phnum; i++) { const Elf_Phdr *phdr = &info->dlpi_phdr[i]; if (phdr->p_type == PT_LOAD) { uptr cur_beg = info->dlpi_addr + phdr->p_vaddr; uptr cur_end = cur_beg + phdr->p_memsz; bool executable = phdr->p_flags & PF_X; - cur_module->addAddressRange(cur_beg, cur_end, executable); + cur_module.addAddressRange(cur_beg, cur_end, executable); } } + data->modules->push_back(cur_module); return 0; } @@ -453,8 +456,8 @@ extern "C" __attribute__((weak)) int dl_iterate_phdr( int (*)(struct dl_phdr_info *, size_t, void *), void *); #endif -uptr GetListOfModules(LoadedModule *modules, uptr max_modules, - string_predicate_t filter) { +void ListOfModules::init() { + clear(); #if SANITIZER_ANDROID && __ANDROID_API__ <= 22 u32 api_level = AndroidGetApiLevel(); // Fall back to /proc/maps if dl_iterate_phdr is unavailable or broken. @@ -462,13 +465,12 @@ uptr GetListOfModules(LoadedModule *modules, uptr max_modules, // both K and L (and future) Android releases. if (api_level <= ANDROID_LOLLIPOP_MR1) { // L or earlier MemoryMappingLayout memory_mapping(false); - return memory_mapping.DumpListOfModules(modules, max_modules, filter); + memory_mapping.DumpListOfModules(&modules_); + return; } #endif - CHECK(modules); - DlIteratePhdrData data = {modules, 0, true, max_modules, filter}; + DlIteratePhdrData data = {&modules_, true}; dl_iterate_phdr(dl_iterate_phdr_cb, &data); - return data.current_n; } // getrusage does not give us the current RSS, only the max RSS. @@ -519,19 +521,20 @@ uptr GetRSS() { static atomic_uint8_t android_log_initialized; void AndroidLogInit() { + openlog(GetProcessName(), 0, LOG_USER); atomic_store(&android_log_initialized, 1, memory_order_release); } -static bool IsSyslogAvailable() { +static bool ShouldLogAfterPrintf() { return atomic_load(&android_log_initialized, memory_order_acquire); } #else void AndroidLogInit() {} -static bool IsSyslogAvailable() { return true; } +static bool ShouldLogAfterPrintf() { return true; } #endif // SANITIZER_ANDROID -static void WriteOneLineToSyslog(const char *s) { +void WriteOneLineToSyslog(const char *s) { #if SANITIZER_ANDROID &&__ANDROID_API__ < 21 __android_log_write(ANDROID_LOG_INFO, NULL, s); #else @@ -539,24 +542,11 @@ static void WriteOneLineToSyslog(const char *s) { #endif } -void WriteToSyslog(const char *buffer) { - if (!IsSyslogAvailable()) - return; - char *copy = internal_strdup(buffer); - char *p = copy; - char *q; - // syslog, at least on Android, has an implicit message length limit. - // Print one line at a time. - do { - q = internal_strchr(p, '\n'); - if (q) - *q = '\0'; - WriteOneLineToSyslog(p); - if (q) - p = q + 1; - } while (q); - InternalFree(copy); +void LogMessageOnPrintf(const char *str) { + if (common_flags()->log_to_syslog && ShouldLogAfterPrintf()) + WriteToSyslog(str); } + #endif // SANITIZER_LINUX } // namespace __sanitizer diff --git a/libsanitizer/sanitizer_common/sanitizer_linux_mips64.S b/libsanitizer/sanitizer_common/sanitizer_linux_mips64.S new file mode 100644 index 000000000000..0b76f3a473a2 --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_linux_mips64.S @@ -0,0 +1,21 @@ +// This file is dual licensed under the MIT and the University of Illinois Open +// Avoid being marked as needing an executable stack: +#if defined(__linux__) && defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif + +// Further contents are mips64 only: +#if defined(__linux__) && defined(__mips64) + +.section .text +.set noreorder +.globl internal_sigreturn +.type internal_sigreturn, @function +internal_sigreturn: + + li $v0,5211 // #5211 is for SYS_rt_sigreturn + syscall + +.size internal_sigreturn, .-internal_sigreturn + +#endif // defined(__linux__) && defined(__mips64) diff --git a/libsanitizer/sanitizer_common/sanitizer_linux_s390.cc b/libsanitizer/sanitizer_common/sanitizer_linux_s390.cc new file mode 100644 index 000000000000..c7d647528aba --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_linux_s390.cc @@ -0,0 +1,189 @@ +//===-- sanitizer_linux_s390.cc -------------------------------------------===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is shared between AddressSanitizer and ThreadSanitizer +// run-time libraries and implements s390-linux-specific functions from +// sanitizer_libc.h. +//===----------------------------------------------------------------------===// + +#include "sanitizer_platform.h" + +#if SANITIZER_LINUX && SANITIZER_S390 + +#include "sanitizer_libc.h" +#include "sanitizer_linux.h" + +#include <errno.h> +#include <sys/syscall.h> +#include <sys/utsname.h> +#include <unistd.h> + +namespace __sanitizer { + +// --------------- sanitizer_libc.h +uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd, + OFF_T offset) { + struct s390_mmap_params { + unsigned long addr; + unsigned long length; + unsigned long prot; + unsigned long flags; + unsigned long fd; + unsigned long offset; + } params = { + (unsigned long)addr, + (unsigned long)length, + (unsigned long)prot, + (unsigned long)flags, + (unsigned long)fd, +# ifdef __s390x__ + (unsigned long)offset, +# else + (unsigned long)(offset / 4096), +# endif + }; +# ifdef __s390x__ + return syscall(__NR_mmap, ¶ms); +# else + return syscall(__NR_mmap2, ¶ms); +# endif +} + +uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, + int *parent_tidptr, void *newtls, int *child_tidptr) { + if (!fn || !child_stack) + return -EINVAL; + CHECK_EQ(0, (uptr)child_stack % 16); + // Minimum frame size. +#ifdef __s390x__ + child_stack = (char *)child_stack - 160; +#else + child_stack = (char *)child_stack - 96; +#endif + // Terminate unwind chain. + ((unsigned long *)child_stack)[0] = 0; + // And pass parameters. + ((unsigned long *)child_stack)[1] = (uptr)fn; + ((unsigned long *)child_stack)[2] = (uptr)arg; + register long res __asm__("r2"); + register void *__cstack __asm__("r2") = child_stack; + register int __flags __asm__("r3") = flags; + register int * __ptidptr __asm__("r4") = parent_tidptr; + register int * __ctidptr __asm__("r5") = child_tidptr; + register void * __newtls __asm__("r6") = newtls; + + __asm__ __volatile__( + /* Clone. */ + "svc %1\n" + + /* if (%r2 != 0) + * return; + */ +#ifdef __s390x__ + "cghi %%r2, 0\n" +#else + "chi %%r2, 0\n" +#endif + "jne 1f\n" + + /* Call "fn(arg)". */ +#ifdef __s390x__ + "lmg %%r1, %%r2, 8(%%r15)\n" +#else + "lm %%r1, %%r2, 4(%%r15)\n" +#endif + "basr %%r14, %%r1\n" + + /* Call _exit(%r2). */ + "svc %2\n" + + /* Return to parent. */ + "1:\n" + : "=r" (res) + : "i"(__NR_clone), "i"(__NR_exit), + "r"(__cstack), + "r"(__flags), + "r"(__ptidptr), + "r"(__ctidptr), + "r"(__newtls) + : "memory", "cc"); + return res; +} + +#if SANITIZER_S390_64 +static bool FixedCVE_2016_2143() { + // Try to determine if the running kernel has a fix for CVE-2016-2143, + // return false if in doubt (better safe than sorry). Distros may want to + // adjust this for their own kernels. + struct utsname buf; + unsigned int major, minor, patch = 0; + // This should never fail, but just in case... + if (uname(&buf)) + return false; + char *ptr = buf.release; + major = internal_simple_strtoll(ptr, &ptr, 10); + // At least first 2 should be matched. + if (ptr[0] != '.') + return false; + minor = internal_simple_strtoll(ptr+1, &ptr, 10); + // Third is optional. + if (ptr[0] == '.') + patch = internal_simple_strtoll(ptr+1, &ptr, 10); + if (major < 3) { + // <3.0 is bad. + return false; + } else if (major == 3) { + // 3.2.79+ is OK. + if (minor == 2 && patch >= 79) + return true; + // 3.12.58+ is OK. + if (minor == 12 && patch >= 58) + return true; + // Otherwise, bad. + return false; + } else if (major == 4) { + // 4.1.21+ is OK. + if (minor == 1 && patch >= 21) + return true; + // 4.4.6+ is OK. + if (minor == 4 && patch >= 6) + return true; + // Otherwise, OK if 4.5+. + return minor >= 5; + } else { + // Linux 5 and up are fine. + return true; + } +} + +void AvoidCVE_2016_2143() { + // Older kernels are affected by CVE-2016-2143 - they will crash hard + // if someone uses 4-level page tables (ie. virtual addresses >= 4TB) + // and fork() in the same process. Unfortunately, sanitizers tend to + // require such addresses. Since this is very likely to crash the whole + // machine (sanitizers themselves use fork() for llvm-symbolizer, for one), + // abort the process at initialization instead. + if (FixedCVE_2016_2143()) + return; + if (GetEnv("SANITIZER_IGNORE_CVE_2016_2143")) + return; + Report( + "ERROR: Your kernel seems to be vulnerable to CVE-2016-2143. Using ASan,\n" + "MSan, TSan, DFSan or LSan with such kernel can and will crash your\n" + "machine, or worse.\n" + "\n" + "If you are certain your kernel is not vulnerable (you have compiled it\n" + "yourself, or are using an unrecognized distribution kernel), you can\n" + "override this safety check by exporting SANITIZER_IGNORE_CVE_2016_2143\n" + "with any value.\n"); + Die(); +} +#endif + +} // namespace __sanitizer + +#endif // SANITIZER_LINUX && SANITIZER_S390 diff --git a/libsanitizer/sanitizer_common/sanitizer_linux_x86_64.S b/libsanitizer/sanitizer_common/sanitizer_linux_x86_64.S new file mode 100644 index 000000000000..6b892116ff4c --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_linux_x86_64.S @@ -0,0 +1,23 @@ +// This file is dual licensed under the MIT and the University of Illinois Open +// Avoid being marked as needing an executable stack: +#if defined(__linux__) && defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif + +// Further contents are x86_64-only: +#if defined(__linux__) && defined(__x86_64__) + +#include "../builtins/assembly.h" + +// If the "naked" function attribute were supported for x86 we could +// do this via inline asm. +.text +.balign 4 +DEFINE_COMPILERRT_FUNCTION(internal_sigreturn) + mov $0xf, %eax // 0xf == SYS_rt_sigreturn + mov %rcx, %r10 + syscall + ret // Won't normally reach here. +END_COMPILERRT_FUNCTION(internal_sigreturn) + +#endif // defined(__linux__) && defined(__x86_64__) diff --git a/libsanitizer/sanitizer_common/sanitizer_list.h b/libsanitizer/sanitizer_common/sanitizer_list.h index 9216ede67fb8..190c7e67cf0f 100644 --- a/libsanitizer/sanitizer_common/sanitizer_list.h +++ b/libsanitizer/sanitizer_common/sanitizer_list.h @@ -69,7 +69,9 @@ struct IntrusiveList { } Item *front() { return first_; } + const Item *front() const { return first_; } Item *back() { return last_; } + const Item *back() const { return last_; } void append_front(IntrusiveList<Item> *l) { CHECK_NE(this, l); @@ -114,24 +116,32 @@ struct IntrusiveList { } } - template<class ListTy, class ItemTy> + template<class ItemTy> class IteratorBase { public: - explicit IteratorBase(ListTy *list) - : list_(list), current_(list->first_) { } - ItemTy *next() { - ItemTy *ret = current_; - if (current_) current_ = current_->next; - return ret; + explicit IteratorBase(ItemTy *current) : current_(current) {} + IteratorBase &operator++() { + current_ = current_->next; + return *this; + } + bool operator!=(IteratorBase other) const { + return current_ != other.current_; + } + ItemTy &operator*() { + return *current_; } - bool hasNext() const { return current_ != nullptr; } private: - ListTy *list_; ItemTy *current_; }; - typedef IteratorBase<IntrusiveList<Item>, Item> Iterator; - typedef IteratorBase<const IntrusiveList<Item>, const Item> ConstIterator; + typedef IteratorBase<Item> Iterator; + typedef IteratorBase<const Item> ConstIterator; + + Iterator begin() { return Iterator(first_); } + Iterator end() { return Iterator(0); } + + ConstIterator begin() const { return ConstIterator(first_); } + ConstIterator end() const { return ConstIterator(0); } // private, don't use directly. uptr size_; diff --git a/libsanitizer/sanitizer_common/sanitizer_mac.cc b/libsanitizer/sanitizer_common/sanitizer_mac.cc index 159db76f01e1..4408d1dccb96 100644 --- a/libsanitizer/sanitizer_common/sanitizer_mac.cc +++ b/libsanitizer/sanitizer_common/sanitizer_mac.cc @@ -11,6 +11,7 @@ #include "sanitizer_platform.h" #if SANITIZER_MAC +#include "sanitizer_mac.h" // Use 64-bit inodes in file operations. ASan does not support OS X 10.5, so // the clients will most certainly use 64-bit ones as well. @@ -23,7 +24,6 @@ #include "sanitizer_flags.h" #include "sanitizer_internal_defs.h" #include "sanitizer_libc.h" -#include "sanitizer_mac.h" #include "sanitizer_placement_new.h" #include "sanitizer_platform_limits_posix.h" #include "sanitizer_procmaps.h" @@ -34,6 +34,23 @@ extern char **environ; #endif +#if defined(__has_include) && __has_include(<os/trace.h>) +#define SANITIZER_OS_TRACE 1 +#include <os/trace.h> +#else +#define SANITIZER_OS_TRACE 0 +#endif + +#if !SANITIZER_IOS +#include <crt_externs.h> // for _NSGetArgv and _NSGetEnviron +#else +extern "C" { + extern char ***_NSGetArgv(void); +} +#endif + +#include <asl.h> +#include <dlfcn.h> // for dladdr() #include <errno.h> #include <fcntl.h> #include <libkern/OSAtomic.h> @@ -49,21 +66,45 @@ extern char **environ; #include <sys/stat.h> #include <sys/sysctl.h> #include <sys/types.h> +#include <sys/wait.h> #include <unistd.h> +#include <util.h> + +// From <crt_externs.h>, but we don't have that file on iOS. +extern "C" { + extern char ***_NSGetArgv(void); + extern char ***_NSGetEnviron(void); +} + +// From <mach/mach_vm.h>, but we don't have that file on iOS. +extern "C" { + extern kern_return_t mach_vm_region_recurse( + vm_map_t target_task, + mach_vm_address_t *address, + mach_vm_size_t *size, + natural_t *nesting_depth, + vm_region_recurse_info_t info, + mach_msg_type_number_t *infoCnt); +} namespace __sanitizer { #include "sanitizer_syscall_generic.inc" +// Direct syscalls, don't call libmalloc hooks. +extern "C" void *__mmap(void *addr, size_t len, int prot, int flags, int fildes, + off_t off); +extern "C" int __munmap(void *, size_t); + // ---------------------- sanitizer_libc.h uptr internal_mmap(void *addr, size_t length, int prot, int flags, int fd, u64 offset) { if (fd == -1) fd = VM_MAKE_TAG(VM_MEMORY_ANALYSIS_TOOL); - return (uptr)mmap(addr, length, prot, flags, fd, offset); + return (uptr)__mmap(addr, length, prot, flags, fd, offset); } uptr internal_munmap(void *addr, uptr length) { - return munmap(addr, length); + return __munmap(addr, length); } int internal_mprotect(void *addr, uptr length, int prot) { @@ -129,6 +170,10 @@ void internal__exit(int exitcode) { _exit(exitcode); } +unsigned int internal_sleep(unsigned int seconds) { + return sleep(seconds); +} + uptr internal_getpid() { return getpid(); } @@ -145,9 +190,34 @@ uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set, return sigprocmask(how, set, oldset); } +// Doesn't call pthread_atfork() handlers. +extern "C" pid_t __fork(void); + int internal_fork() { - // TODO(glider): this may call user's pthread_atfork() handlers which is bad. - return fork(); + return __fork(); +} + +int internal_forkpty(int *amaster) { + int master, slave; + if (openpty(&master, &slave, nullptr, nullptr, nullptr) == -1) return -1; + int pid = __fork(); + if (pid == -1) { + close(master); + close(slave); + return -1; + } + if (pid == 0) { + close(master); + if (login_tty(slave) != 0) { + // We already forked, there's not much we can do. Let's quit. + Report("login_tty failed (errno %d)\n", errno); + internal__exit(1); + } + } else { + *amaster = master; + close(slave); + } + return pid; } uptr internal_rename(const char *oldpath, const char *newpath) { @@ -158,6 +228,15 @@ uptr internal_ftruncate(fd_t fd, uptr size) { return ftruncate(fd, size); } +uptr internal_execve(const char *filename, char *const argv[], + char *const envp[]) { + return execve(filename, argv, envp); +} + +uptr internal_waitpid(int pid, int *status, int options) { + return waitpid(pid, status, options); +} + // ----------------- sanitizer_common.h bool FileExists(const char *filename) { struct stat st; @@ -168,7 +247,10 @@ bool FileExists(const char *filename) { } uptr GetTid() { - return reinterpret_cast<uptr>(pthread_self()); + // FIXME: This can potentially get truncated on 32-bit, where uptr is 4 bytes. + uint64_t tid; + pthread_threadid_np(nullptr, &tid); + return tid; } void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, @@ -178,7 +260,7 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, uptr stacksize = pthread_get_stacksize_np(pthread_self()); // pthread_get_stacksize_np() returns an incorrect stack size for the main // thread on Mavericks. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=261 + // https://github.com/google/sanitizers/issues/261 if ((GetMacosVersion() >= MACOS_VERSION_MAVERICKS) && at_initialization && stacksize == (1 << 19)) { struct rlimit rl; @@ -289,7 +371,7 @@ void InitTlsSize() { void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, uptr *tls_addr, uptr *tls_size) { -#ifndef SANITIZER_GO +#if !SANITIZER_GO uptr stack_top, stack_bottom; GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom); *stk_addr = stack_bottom; @@ -304,13 +386,18 @@ void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, #endif } -uptr GetListOfModules(LoadedModule *modules, uptr max_modules, - string_predicate_t filter) { +void ListOfModules::init() { + clear(); MemoryMappingLayout memory_mapping(false); - return memory_mapping.DumpListOfModules(modules, max_modules, filter); + memory_mapping.DumpListOfModules(&modules_); } -bool IsDeadlySignal(int signum) { +bool IsHandledDeadlySignal(int signum) { + if ((SANITIZER_WATCHOS || SANITIZER_TVOS) && !(SANITIZER_IOSSIM)) + // Handling fatal signals on watchOS and tvOS devices is disallowed. + return false; + if (common_flags()->handle_abort && signum == SIGABRT) + return true; return (signum == SIGSEGV || signum == SIGBUS) && common_flags()->handle_segv; } @@ -382,6 +469,70 @@ void *internal_start_thread(void(*func)(void *arg), void *arg) { void internal_join_thread(void *th) { pthread_join((pthread_t)th, 0); } +#if !SANITIZER_GO +static BlockingMutex syslog_lock(LINKER_INITIALIZED); +#endif + +void WriteOneLineToSyslog(const char *s) { +#if !SANITIZER_GO + syslog_lock.CheckLocked(); + asl_log(nullptr, nullptr, ASL_LEVEL_ERR, "%s", s); +#endif +} + +void LogMessageOnPrintf(const char *str) { + // Log all printf output to CrashLog. + if (common_flags()->abort_on_error) + CRAppendCrashLogMessage(str); +} + +void LogFullErrorReport(const char *buffer) { +#if !SANITIZER_GO + // Log with os_trace. This will make it into the crash log. +#if SANITIZER_OS_TRACE + if (GetMacosVersion() >= MACOS_VERSION_YOSEMITE) { + // os_trace requires the message (format parameter) to be a string literal. + if (internal_strncmp(SanitizerToolName, "AddressSanitizer", + sizeof("AddressSanitizer") - 1) == 0) + os_trace("Address Sanitizer reported a failure."); + else if (internal_strncmp(SanitizerToolName, "UndefinedBehaviorSanitizer", + sizeof("UndefinedBehaviorSanitizer") - 1) == 0) + os_trace("Undefined Behavior Sanitizer reported a failure."); + else if (internal_strncmp(SanitizerToolName, "ThreadSanitizer", + sizeof("ThreadSanitizer") - 1) == 0) + os_trace("Thread Sanitizer reported a failure."); + else + os_trace("Sanitizer tool reported a failure."); + + if (common_flags()->log_to_syslog) + os_trace("Consult syslog for more information."); + } +#endif + + // Log to syslog. + // The logging on OS X may call pthread_create so we need the threading + // environment to be fully initialized. Also, this should never be called when + // holding the thread registry lock since that may result in a deadlock. If + // the reporting thread holds the thread registry mutex, and asl_log waits + // for GCD to dispatch a new thread, the process will deadlock, because the + // pthread_create wrapper needs to acquire the lock as well. + BlockingMutexLock l(&syslog_lock); + if (common_flags()->log_to_syslog) + WriteToSyslog(buffer); + + // The report is added to CrashLog as part of logging all of Printf output. +#endif +} + +SignalContext::WriteFlag SignalContext::GetWriteFlag(void *context) { +#if defined(__x86_64__) || defined(__i386__) + ucontext_t *ucontext = static_cast<ucontext_t*>(context); + return ucontext->uc_mcontext->__es.__err & 2 /*T_PF_WRITE*/ ? WRITE : READ; +#else + return UNKNOWN; +#endif +} + void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { ucontext_t *ucontext = (ucontext_t*)context; # if defined(__aarch64__) @@ -409,6 +560,237 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { # endif } +#if !SANITIZER_GO +static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES"; +LowLevelAllocator allocator_for_env; + +// Change the value of the env var |name|, leaking the original value. +// If |name_value| is NULL, the variable is deleted from the environment, +// otherwise the corresponding "NAME=value" string is replaced with +// |name_value|. +void LeakyResetEnv(const char *name, const char *name_value) { + char **env = GetEnviron(); + uptr name_len = internal_strlen(name); + while (*env != 0) { + uptr len = internal_strlen(*env); + if (len > name_len) { + const char *p = *env; + if (!internal_memcmp(p, name, name_len) && p[name_len] == '=') { + // Match. + if (name_value) { + // Replace the old value with the new one. + *env = const_cast<char*>(name_value); + } else { + // Shift the subsequent pointers back. + char **del = env; + do { + del[0] = del[1]; + } while (*del++); + } + } + } + env++; + } +} + +SANITIZER_WEAK_CXX_DEFAULT_IMPL +bool ReexecDisabled() { + return false; +} + +extern "C" SANITIZER_WEAK_ATTRIBUTE double dyldVersionNumber; +static const double kMinDyldVersionWithAutoInterposition = 360.0; + +bool DyldNeedsEnvVariable() { + // Although sanitizer support was added to LLVM on OS X 10.7+, GCC users + // still may want use them on older systems. On older Darwin platforms, dyld + // doesn't export dyldVersionNumber symbol and we simply return true. + if (!&dyldVersionNumber) return true; + // If running on OS X 10.11+ or iOS 9.0+, dyld will interpose even if + // DYLD_INSERT_LIBRARIES is not set. However, checking OS version via + // GetMacosVersion() doesn't work for the simulator. Let's instead check + // `dyldVersionNumber`, which is exported by dyld, against a known version + // number from the first OS release where this appeared. + return dyldVersionNumber < kMinDyldVersionWithAutoInterposition; +} + +void MaybeReexec() { + if (ReexecDisabled()) return; + + // Make sure the dynamic runtime library is preloaded so that the + // wrappers work. If it is not, set DYLD_INSERT_LIBRARIES and re-exec + // ourselves. + Dl_info info; + RAW_CHECK(dladdr((void*)((uptr)&__sanitizer_report_error_summary), &info)); + char *dyld_insert_libraries = + const_cast<char*>(GetEnv(kDyldInsertLibraries)); + uptr old_env_len = dyld_insert_libraries ? + internal_strlen(dyld_insert_libraries) : 0; + uptr fname_len = internal_strlen(info.dli_fname); + const char *dylib_name = StripModuleName(info.dli_fname); + uptr dylib_name_len = internal_strlen(dylib_name); + + bool lib_is_in_env = dyld_insert_libraries && + internal_strstr(dyld_insert_libraries, dylib_name); + if (DyldNeedsEnvVariable() && !lib_is_in_env) { + // DYLD_INSERT_LIBRARIES is not set or does not contain the runtime + // library. + InternalScopedString program_name(1024); + uint32_t buf_size = program_name.size(); + _NSGetExecutablePath(program_name.data(), &buf_size); + char *new_env = const_cast<char*>(info.dli_fname); + if (dyld_insert_libraries) { + // Append the runtime dylib name to the existing value of + // DYLD_INSERT_LIBRARIES. + new_env = (char*)allocator_for_env.Allocate(old_env_len + fname_len + 2); + internal_strncpy(new_env, dyld_insert_libraries, old_env_len); + new_env[old_env_len] = ':'; + // Copy fname_len and add a trailing zero. + internal_strncpy(new_env + old_env_len + 1, info.dli_fname, + fname_len + 1); + // Ok to use setenv() since the wrappers don't depend on the value of + // asan_inited. + setenv(kDyldInsertLibraries, new_env, /*overwrite*/1); + } else { + // Set DYLD_INSERT_LIBRARIES equal to the runtime dylib name. + setenv(kDyldInsertLibraries, info.dli_fname, /*overwrite*/0); + } + VReport(1, "exec()-ing the program with\n"); + VReport(1, "%s=%s\n", kDyldInsertLibraries, new_env); + VReport(1, "to enable wrappers.\n"); + execv(program_name.data(), *_NSGetArgv()); + + // We get here only if execv() failed. + Report("ERROR: The process is launched without DYLD_INSERT_LIBRARIES, " + "which is required for the sanitizer to work. We tried to set the " + "environment variable and re-execute itself, but execv() failed, " + "possibly because of sandbox restrictions. Make sure to launch the " + "executable with:\n%s=%s\n", kDyldInsertLibraries, new_env); + RAW_CHECK("execv failed" && 0); + } + + // Verify that interceptors really work. We'll use dlsym to locate + // "pthread_create", if interceptors are working, it should really point to + // "wrap_pthread_create" within our own dylib. + Dl_info info_pthread_create; + void *dlopen_addr = dlsym(RTLD_DEFAULT, "pthread_create"); + RAW_CHECK(dladdr(dlopen_addr, &info_pthread_create)); + if (internal_strcmp(info.dli_fname, info_pthread_create.dli_fname) != 0) { + Report( + "ERROR: Interceptors are not working. This may be because %s is " + "loaded too late (e.g. via dlopen). Please launch the executable " + "with:\n%s=%s\n", + SanitizerToolName, kDyldInsertLibraries, info.dli_fname); + RAW_CHECK("interceptors not installed" && 0); + } + + if (!lib_is_in_env) + return; + + // DYLD_INSERT_LIBRARIES is set and contains the runtime library. Let's remove + // the dylib from the environment variable, because interceptors are installed + // and we don't want our children to inherit the variable. + + uptr env_name_len = internal_strlen(kDyldInsertLibraries); + // Allocate memory to hold the previous env var name, its value, the '=' + // sign and the '\0' char. + char *new_env = (char*)allocator_for_env.Allocate( + old_env_len + 2 + env_name_len); + RAW_CHECK(new_env); + internal_memset(new_env, '\0', old_env_len + 2 + env_name_len); + internal_strncpy(new_env, kDyldInsertLibraries, env_name_len); + new_env[env_name_len] = '='; + char *new_env_pos = new_env + env_name_len + 1; + + // Iterate over colon-separated pieces of |dyld_insert_libraries|. + char *piece_start = dyld_insert_libraries; + char *piece_end = NULL; + char *old_env_end = dyld_insert_libraries + old_env_len; + do { + if (piece_start[0] == ':') piece_start++; + piece_end = internal_strchr(piece_start, ':'); + if (!piece_end) piece_end = dyld_insert_libraries + old_env_len; + if ((uptr)(piece_start - dyld_insert_libraries) > old_env_len) break; + uptr piece_len = piece_end - piece_start; + + char *filename_start = + (char *)internal_memrchr(piece_start, '/', piece_len); + uptr filename_len = piece_len; + if (filename_start) { + filename_start += 1; + filename_len = piece_len - (filename_start - piece_start); + } else { + filename_start = piece_start; + } + + // If the current piece isn't the runtime library name, + // append it to new_env. + if ((dylib_name_len != filename_len) || + (internal_memcmp(filename_start, dylib_name, dylib_name_len) != 0)) { + if (new_env_pos != new_env + env_name_len + 1) { + new_env_pos[0] = ':'; + new_env_pos++; + } + internal_strncpy(new_env_pos, piece_start, piece_len); + new_env_pos += piece_len; + } + // Move on to the next piece. + piece_start = piece_end; + } while (piece_start < old_env_end); + + // Can't use setenv() here, because it requires the allocator to be + // initialized. + // FIXME: instead of filtering DYLD_INSERT_LIBRARIES here, do it in + // a separate function called after InitializeAllocator(). + if (new_env_pos == new_env + env_name_len + 1) new_env = NULL; + LeakyResetEnv(kDyldInsertLibraries, new_env); +} +#endif // SANITIZER_GO + +char **GetArgv() { + return *_NSGetArgv(); +} + +uptr FindAvailableMemoryRange(uptr shadow_size, + uptr alignment, + uptr left_padding) { + typedef vm_region_submap_short_info_data_64_t RegionInfo; + enum { kRegionInfoSize = VM_REGION_SUBMAP_SHORT_INFO_COUNT_64 }; + // Start searching for available memory region past PAGEZERO, which is + // 4KB on 32-bit and 4GB on 64-bit. + mach_vm_address_t start_address = + (SANITIZER_WORDSIZE == 32) ? 0x000000001000 : 0x000100000000; + + mach_vm_address_t address = start_address; + mach_vm_address_t free_begin = start_address; + kern_return_t kr = KERN_SUCCESS; + while (kr == KERN_SUCCESS) { + mach_vm_size_t vmsize = 0; + natural_t depth = 0; + RegionInfo vminfo; + mach_msg_type_number_t count = kRegionInfoSize; + kr = mach_vm_region_recurse(mach_task_self(), &address, &vmsize, &depth, + (vm_region_info_t)&vminfo, &count); + if (free_begin != address) { + // We found a free region [free_begin..address-1]. + uptr shadow_address = RoundUpTo((uptr)free_begin + left_padding, + alignment); + if (shadow_address + shadow_size < (uptr)address) { + return shadow_address; + } + } + // Move to the next region. + address += vmsize; + free_begin = address; + } + + // We looked at all free regions and could not find one large enough. + return 0; +} + +// FIXME implement on this platform. +void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) { } + } // namespace __sanitizer #endif // SANITIZER_MAC diff --git a/libsanitizer/sanitizer_common/sanitizer_mac.h b/libsanitizer/sanitizer_common/sanitizer_mac.h index 2aac83e3657c..4bea069189f9 100644 --- a/libsanitizer/sanitizer_common/sanitizer_mac.h +++ b/libsanitizer/sanitizer_common/sanitizer_mac.h @@ -11,6 +11,7 @@ #ifndef SANITIZER_MAC_H #define SANITIZER_MAC_H +#include "sanitizer_common.h" #include "sanitizer_platform.h" #if SANITIZER_MAC #include "sanitizer_posix.h" @@ -35,5 +36,22 @@ char **GetEnviron(); } // namespace __sanitizer +extern "C" { +static char __crashreporter_info_buff__[__sanitizer::kErrorMessageBufferSize] = + {}; +static const char *__crashreporter_info__ __attribute__((__used__)) = + &__crashreporter_info_buff__[0]; +asm(".desc ___crashreporter_info__, 0x10"); +} // extern "C" + +namespace __sanitizer { +static BlockingMutex crashreporter_info_mutex(LINKER_INITIALIZED); + +INLINE void CRAppendCrashLogMessage(const char *msg) { + BlockingMutexLock l(&crashreporter_info_mutex); + internal_strlcat(__crashreporter_info_buff__, msg, + sizeof(__crashreporter_info_buff__)); } +} // namespace __sanitizer + #endif // SANITIZER_MAC #endif // SANITIZER_MAC_H diff --git a/libsanitizer/sanitizer_common/sanitizer_malloc_mac.inc b/libsanitizer/sanitizer_common/sanitizer_malloc_mac.inc index 6ed0f69de974..93fb19e922cd 100644 --- a/libsanitizer/sanitizer_common/sanitizer_malloc_mac.inc +++ b/libsanitizer/sanitizer_common/sanitizer_malloc_mac.inc @@ -25,7 +25,7 @@ #include "sanitizer_common/sanitizer_mac.h" // Similar code is used in Google Perftools, -// http://code.google.com/p/google-perftools. +// https://github.com/gperftools/gperftools. static malloc_zone_t sanitizer_zone; @@ -54,7 +54,7 @@ INTERCEPTOR(malloc_zone_t *, malloc_default_zone, void) { INTERCEPTOR(malloc_zone_t *, malloc_default_purgeable_zone, void) { // FIXME: ASan should support purgeable allocations. - // https://code.google.com/p/address-sanitizer/issues/detail?id=139 + // https://github.com/google/sanitizers/issues/139 COMMON_MALLOC_ENTER(); return &sanitizer_zone; } @@ -182,28 +182,18 @@ void *__sanitizer_mz_valloc(malloc_zone_t *zone, size_t size) { return p; } -#define GET_ZONE_FOR_PTR(ptr) \ - malloc_zone_t *zone_ptr = malloc_zone_from_ptr(ptr); \ - const char *zone_name = (zone_ptr == 0) ? 0 : zone_ptr->zone_name - -void ALWAYS_INLINE free_common(void *context, void *ptr) { - if (!ptr) return; - // FIXME: need to retire this flag. - if (!COMMON_MALLOC_IGNORE_INVALID_FREE) { - COMMON_MALLOC_FREE(ptr); - } else { - GET_ZONE_FOR_PTR(ptr); - COMMON_MALLOC_REPORT_FREE_UNALLOCATED(ptr, zone_ptr, zone_name); - } -} - // TODO(glider): the allocation callbacks need to be refactored. extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_mz_free(malloc_zone_t *zone, void *ptr) { - free_common(zone, ptr); + if (!ptr) return; + COMMON_MALLOC_FREE(ptr); } +#define GET_ZONE_FOR_PTR(ptr) \ + malloc_zone_t *zone_ptr = malloc_zone_from_ptr(ptr); \ + const char *zone_name = (zone_ptr == 0) ? 0 : zone_ptr->zone_name + extern "C" SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_mz_realloc(malloc_zone_t *zone, void *ptr, size_t new_size) { diff --git a/libsanitizer/sanitizer_common/sanitizer_platform.h b/libsanitizer/sanitizer_common/sanitizer_platform.h index 7d0ff2896e95..428709d55ec3 100644 --- a/libsanitizer/sanitizer_common/sanitizer_platform.h +++ b/libsanitizer/sanitizer_common/sanitizer_platform.h @@ -47,12 +47,30 @@ # define SANITIZER_IOSSIM 0 #endif +#if defined(__APPLE__) && TARGET_OS_IPHONE && TARGET_OS_WATCH +# define SANITIZER_WATCHOS 1 +#else +# define SANITIZER_WATCHOS 0 +#endif + +#if defined(__APPLE__) && TARGET_OS_IPHONE && TARGET_OS_TV +# define SANITIZER_TVOS 1 +#else +# define SANITIZER_TVOS 0 +#endif + #if defined(_WIN32) # define SANITIZER_WINDOWS 1 #else # define SANITIZER_WINDOWS 0 #endif +#if defined(_WIN64) +# define SANITIZER_WINDOWS64 1 +#else +# define SANITIZER_WINDOWS64 0 +#endif + #if defined(__ANDROID__) # define SANITIZER_ANDROID 1 #else @@ -79,14 +97,67 @@ # define SANITIZER_X32 0 #endif -// VMA size definition for architecture that support multiple sizes. -// AArch64 has 3 VMA sizes: 39, 42 and 48. -#if !defined(SANITIZER_AARCH64_VMA) -# define SANITIZER_AARCH64_VMA 39 +#if defined(__mips__) +# define SANITIZER_MIPS 1 +# if defined(__mips64) +# define SANITIZER_MIPS32 0 +# define SANITIZER_MIPS64 1 +# else +# define SANITIZER_MIPS32 1 +# define SANITIZER_MIPS64 0 +# endif #else -# if SANITIZER_AARCH64_VMA != 39 && SANITIZER_AARCH64_VMA != 42 -# error "invalid SANITIZER_AARCH64_VMA size" +# define SANITIZER_MIPS 0 +# define SANITIZER_MIPS32 0 +# define SANITIZER_MIPS64 0 +#endif + +#if defined(__s390__) +# define SANITIZER_S390 1 +# if defined(__s390x__) +# define SANITIZER_S390_31 0 +# define SANITIZER_S390_64 1 +# else +# define SANITIZER_S390_31 1 +# define SANITIZER_S390_64 0 +# endif +#else +# define SANITIZER_S390 0 +# define SANITIZER_S390_31 0 +# define SANITIZER_S390_64 0 +#endif + +#if defined(__powerpc__) +# define SANITIZER_PPC 1 +# if defined(__powerpc64__) +# define SANITIZER_PPC32 0 +# define SANITIZER_PPC64 1 +// 64-bit PPC has two ABIs (v1 and v2). The old powerpc64 target is +// big-endian, and uses v1 ABI (known for its function descriptors), +// while the new powerpc64le target is little-endian and uses v2. +// In theory, you could convince gcc to compile for their evil twins +// (eg. big-endian v2), but you won't find such combinations in the wild +// (it'd require bootstrapping a whole system, which would be quite painful +// - there's no target triple for that). LLVM doesn't support them either. +# if _CALL_ELF == 2 +# define SANITIZER_PPC64V1 0 +# define SANITIZER_PPC64V2 1 +# else +# define SANITIZER_PPC64V1 1 +# define SANITIZER_PPC64V2 0 +# endif +# else +# define SANITIZER_PPC32 1 +# define SANITIZER_PPC64 0 +# define SANITIZER_PPC64V1 0 +# define SANITIZER_PPC64V2 0 # endif +#else +# define SANITIZER_PPC 0 +# define SANITIZER_PPC32 0 +# define SANITIZER_PPC64 0 +# define SANITIZER_PPC64V1 0 +# define SANITIZER_PPC64V2 0 #endif // By default we allow to use SizeClassAllocator64 on 64-bit platform. @@ -95,7 +166,9 @@ // For such platforms build this code with -DSANITIZER_CAN_USE_ALLOCATOR64=0 or // change the definition of SANITIZER_CAN_USE_ALLOCATOR64 here. #ifndef SANITIZER_CAN_USE_ALLOCATOR64 -# if defined(__mips64) || defined(__aarch64__) +# if SANITIZER_ANDROID && defined(__aarch64__) +# define SANITIZER_CAN_USE_ALLOCATOR64 1 +# elif defined(__mips64) || defined(__aarch64__) # define SANITIZER_CAN_USE_ALLOCATOR64 0 # else # define SANITIZER_CAN_USE_ALLOCATOR64 (SANITIZER_WORDSIZE == 64) @@ -107,6 +180,8 @@ // will still work but will consume more memory for TwoLevelByteMap. #if defined(__mips__) # define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 40) +#elif defined(__aarch64__) +# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 48) #else # define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47) #endif @@ -135,10 +210,8 @@ #define SANITIZER_USES_UID16_SYSCALLS 0 #endif -#if defined(__mips__) || (defined(__aarch64__) && SANITIZER_AARCH64_VMA == 39) +#if defined(__mips__) # define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 10) -#elif defined(__aarch64__) && SANITIZER_AARCH64_VMA == 42 -# define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 11) #else # define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 12) #endif @@ -160,4 +233,22 @@ # define MSC_PREREQ(version) 0 #endif +#if defined(__arm64__) && SANITIZER_IOS +# define SANITIZER_NON_UNIQUE_TYPEINFO 1 +#else +# define SANITIZER_NON_UNIQUE_TYPEINFO 0 +#endif + +// On linux, some architectures had an ABI transition from 64-bit long double +// (ie. same as double) to 128-bit long double. On those, glibc symbols +// involving long doubles come in two versions, and we need to pass the +// correct one to dlvsym when intercepting them. +#if SANITIZER_LINUX && (SANITIZER_S390 || SANITIZER_PPC32 || SANITIZER_PPC64V1) +#define SANITIZER_NLDBL_VERSION "GLIBC_2.4" +#endif + +#if SANITIZER_GO == 0 +# define SANITIZER_GO 0 +#endif + #endif // SANITIZER_PLATFORM_H diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h b/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h index 040e030e088b..2a8860581870 100644 --- a/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h +++ b/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h @@ -27,6 +27,12 @@ # define SI_LINUX_NOT_ANDROID 0 #endif +#if SANITIZER_ANDROID +# define SI_ANDROID 1 +#else +# define SI_ANDROID 0 +#endif + #if SANITIZER_FREEBSD # define SI_FREEBSD 1 #else @@ -41,8 +47,10 @@ #if SANITIZER_MAC # define SI_MAC 1 +# define SI_NOT_MAC 0 #else # define SI_MAC 0 +# define SI_NOT_MAC 1 #endif #if SANITIZER_IOS @@ -51,14 +59,30 @@ # define SI_IOS 0 #endif +#if !SANITIZER_WINDOWS && !SANITIZER_MAC +# define SI_UNIX_NOT_MAC 1 +#else +# define SI_UNIX_NOT_MAC 0 +#endif + +#define SANITIZER_INTERCEPT_STRLEN 1 +#define SANITIZER_INTERCEPT_STRNLEN SI_NOT_MAC #define SANITIZER_INTERCEPT_STRCMP 1 #define SANITIZER_INTERCEPT_STRSTR 1 #define SANITIZER_INTERCEPT_STRCASESTR SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_STRCHR 1 +#define SANITIZER_INTERCEPT_STRCHRNUL SI_UNIX_NOT_MAC +#define SANITIZER_INTERCEPT_STRRCHR 1 #define SANITIZER_INTERCEPT_STRSPN 1 #define SANITIZER_INTERCEPT_STRPBRK 1 #define SANITIZER_INTERCEPT_TEXTDOMAIN SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_STRCASECMP SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_MEMSET 1 +#define SANITIZER_INTERCEPT_MEMMOVE 1 +#define SANITIZER_INTERCEPT_MEMCPY 1 #define SANITIZER_INTERCEPT_MEMCMP 1 +// FIXME: enable memmem on Windows. +#define SANITIZER_INTERCEPT_MEMMEM SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_MEMCHR 1 #define SANITIZER_INTERCEPT_MEMRCHR SI_FREEBSD || SI_LINUX @@ -123,15 +147,21 @@ #define SANITIZER_INTERCEPT_ACCEPT4 SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_MODF SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_RECVMSG SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_SENDMSG SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_GETPEERNAME SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_IOCTL SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_INET_ATON SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_SYSINFO SI_LINUX #define SANITIZER_INTERCEPT_READDIR SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_READDIR64 SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_PTRACE SI_LINUX_NOT_ANDROID && \ +#if SI_LINUX_NOT_ANDROID && \ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \ - defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__)) + defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \ + defined(__s390__)) +#define SANITIZER_INTERCEPT_PTRACE 1 +#else +#define SANITIZER_INTERCEPT_PTRACE 0 +#endif #define SANITIZER_INTERCEPT_SETLOCALE SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_GETCWD SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_GET_CURRENT_DIR_NAME SI_LINUX_NOT_ANDROID @@ -203,6 +233,7 @@ #define SANITIZER_INTERCEPT_PTHREAD_BARRIERATTR_GETPSHARED SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_TMPNAM SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_TMPNAM_R SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_TTYNAME_R SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_TEMPNAM SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_SINCOS SI_LINUX #define SANITIZER_INTERCEPT_REMQUO SI_NOT_WINDOWS @@ -235,7 +266,11 @@ #define SANITIZER_INTERCEPT_IF_INDEXTONAME \ SI_FREEBSD || SI_LINUX_NOT_ANDROID || SI_MAC #define SANITIZER_INTERCEPT_CAPGET SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT_AEABI_MEM SI_LINUX && defined(__arm__) +#if SI_LINUX && defined(__arm__) +#define SANITIZER_INTERCEPT_AEABI_MEM 1 +#else +#define SANITIZER_INTERCEPT_AEABI_MEM 0 +#endif #define SANITIZER_INTERCEPT___BZERO SI_MAC #define SANITIZER_INTERCEPT_FTIME !SI_FREEBSD && SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_XDR SI_LINUX_NOT_ANDROID @@ -247,8 +282,12 @@ #define SANITIZER_INTERCEPT_OBSTACK SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_FFLUSH SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_FCLOSE SI_NOT_WINDOWS + +#ifndef SANITIZER_INTERCEPT_DLOPEN_DLCLOSE #define SANITIZER_INTERCEPT_DLOPEN_DLCLOSE \ SI_FREEBSD || SI_LINUX_NOT_ANDROID || SI_MAC +#endif + #define SANITIZER_INTERCEPT_GETPASS SI_LINUX_NOT_ANDROID || SI_MAC #define SANITIZER_INTERCEPT_TIMERFD SI_LINUX_NOT_ANDROID @@ -258,7 +297,17 @@ #define SANITIZER_INTERCEPT_PTHREAD_SETCANCEL SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_MINCORE SI_LINUX #define SANITIZER_INTERCEPT_PROCESS_VM_READV SI_LINUX +#define SANITIZER_INTERCEPT_CTERMID SI_LINUX || SI_MAC || SI_FREEBSD +#define SANITIZER_INTERCEPT_CTERMID_R SI_MAC || SI_FREEBSD #define SANITIZER_INTERCEPTOR_HOOKS SI_LINUX +#define SANITIZER_INTERCEPT_RECV_RECVFROM SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_SEND_SENDTO SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_EVENTFD_READ_WRITE SI_LINUX +#define SANITIZER_INTERCEPT_STAT (SI_FREEBSD || SI_MAC || SI_ANDROID) +#define SANITIZER_INTERCEPT___XSTAT !SANITIZER_INTERCEPT_STAT && SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT___XSTAT64 SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT___LXSTAT SANITIZER_INTERCEPT___XSTAT +#define SANITIZER_INTERCEPT___LXSTAT64 SI_LINUX_NOT_ANDROID #endif // #ifndef SANITIZER_PLATFORM_INTERCEPTORS_H diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_linux.cc b/libsanitizer/sanitizer_common/sanitizer_platform_limits_linux.cc index a1f043250331..edc673019359 100644 --- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_linux.cc +++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_linux.cc @@ -26,7 +26,7 @@ // With old kernels (and even new kernels on powerpc) asm/stat.h uses types that // are not defined anywhere in userspace headers. Fake them. This seems to work // fine with newer headers, too. -#include <asm/posix_types.h> +#include <linux/posix_types.h> #if defined(__x86_64__) || defined(__mips__) #include <sys/stat.h> #else @@ -36,7 +36,6 @@ #define uid_t __kernel_uid_t #define gid_t __kernel_gid_t #define off_t __kernel_off_t -#define time_t __kernel_time_t // This header seems to contain the definitions of _kernel_ stat* structs. #include <asm/stat.h> #undef ino_t @@ -54,6 +53,8 @@ #include <linux/perf_event.h> #endif +using namespace __sanitizer; + namespace __sanitizer { #if !SANITIZER_ANDROID unsigned struct_statfs64_sz = sizeof(struct statfs64); @@ -61,7 +62,7 @@ namespace __sanitizer { } // namespace __sanitizer #if !defined(__powerpc64__) && !defined(__x86_64__) && !defined(__aarch64__)\ - && !defined(__mips__) && !defined(__sparc__) + && !defined(__mips__) && !defined(__s390__) COMPILER_CHECK(struct___old_kernel_stat_sz == sizeof(struct __old_kernel_stat)); #endif diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc index 9866cc9e17ae..7bf76c4e7152 100644 --- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc +++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc @@ -121,6 +121,10 @@ # include <asm/ptrace.h> # ifdef __arm__ typedef struct user_fpregs elf_fpregset_t; +# define ARM_VFPREGS_SIZE_ASAN (32 * 8 /*fpregs*/ + 4 /*fpscr*/) +# if !defined(ARM_VFPREGS_SIZE) +# define ARM_VFPREGS_SIZE ARM_VFPREGS_SIZE_ASAN +# endif # endif # endif # include <semaphore.h> @@ -305,23 +309,28 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); #if SANITIZER_LINUX && !SANITIZER_ANDROID && \ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \ - defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__)) + defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \ + defined(__s390__)) #if defined(__mips64) || defined(__powerpc64__) || defined(__arm__) unsigned struct_user_regs_struct_sz = sizeof(struct pt_regs); unsigned struct_user_fpregs_struct_sz = sizeof(elf_fpregset_t); #elif defined(__aarch64__) unsigned struct_user_regs_struct_sz = sizeof(struct user_pt_regs); unsigned struct_user_fpregs_struct_sz = sizeof(struct user_fpsimd_state); +#elif defined(__s390__) + unsigned struct_user_regs_struct_sz = sizeof(struct _user_regs_struct); + unsigned struct_user_fpregs_struct_sz = sizeof(struct _user_fpregs_struct); #else unsigned struct_user_regs_struct_sz = sizeof(struct user_regs_struct); unsigned struct_user_fpregs_struct_sz = sizeof(struct user_fpregs_struct); #endif // __mips64 || __powerpc64__ || __aarch64__ #if defined(__x86_64) || defined(__mips64) || defined(__powerpc64__) || \ - defined(__aarch64__) || defined(__arm__) + defined(__aarch64__) || defined(__arm__) || defined(__s390__) unsigned struct_user_fpxregs_struct_sz = 0; #else unsigned struct_user_fpxregs_struct_sz = sizeof(struct user_fpxregs_struct); #endif // __x86_64 || __mips64 || __powerpc64__ || __aarch64__ || __arm__ +// || __s390__ #ifdef __arm__ unsigned struct_user_vfpregs_struct_sz = ARM_VFPREGS_SIZE; #else @@ -926,6 +935,8 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); const int si_SEGV_ACCERR = SEGV_ACCERR; } // namespace __sanitizer +using namespace __sanitizer; + COMPILER_CHECK(sizeof(__sanitizer_pthread_attr_t) >= sizeof(pthread_attr_t)); COMPILER_CHECK(sizeof(socklen_t) == sizeof(unsigned)); @@ -1049,8 +1060,15 @@ COMPILER_CHECK(sizeof(__sanitizer_sigaction) == sizeof(struct sigaction)); // Can't write checks for sa_handler and sa_sigaction due to them being // preprocessor macros. CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_mask); +#ifndef __GLIBC_PREREQ +#define __GLIBC_PREREQ(x, y) 0 +#endif +#if !defined(__s390x__) || __GLIBC_PREREQ (2, 20) +// On s390x glibc 2.19 and earlier sa_flags was unsigned long, and sa_resv +// didn't exist. CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_flags); -#if SANITIZER_LINUX +#endif +#if SANITIZER_LINUX && (!SANITIZER_ANDROID || !SANITIZER_MIPS32) CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_restorer); #endif @@ -1121,9 +1139,6 @@ CHECK_SIZE_AND_OFFSET(ipc_perm, uid); CHECK_SIZE_AND_OFFSET(ipc_perm, gid); CHECK_SIZE_AND_OFFSET(ipc_perm, cuid); CHECK_SIZE_AND_OFFSET(ipc_perm, cgid); -#ifndef __GLIBC_PREREQ -#define __GLIBC_PREREQ(x, y) 0 -#endif #if !defined(__aarch64__) || !SANITIZER_LINUX || __GLIBC_PREREQ (2, 21) /* On aarch64 glibc 2.20 and earlier provided incorrect mode field. */ CHECK_SIZE_AND_OFFSET(ipc_perm, mode); @@ -1262,4 +1277,8 @@ CHECK_SIZE_AND_OFFSET(cookie_io_functions_t, close); CHECK_TYPE_SIZE(sem_t); #endif +#if SANITIZER_LINUX && defined(__arm__) +COMPILER_CHECK(ARM_VFPREGS_SIZE == ARM_VFPREGS_SIZE_ASAN); +#endif + #endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_MAC diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h index b6f90eb3a740..17906d303368 100644 --- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h +++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h @@ -18,7 +18,7 @@ #if SANITIZER_FREEBSD // FreeBSD's dlopen() returns a pointer to an Obj_Entry structure that -// incroporates the map structure. +// incorporates the map structure. # define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \ ((link_map*)((handle) == nullptr ? nullptr : ((char*)(handle) + 544))) #else @@ -75,20 +75,16 @@ namespace __sanitizer { const unsigned struct_kernel_stat_sz = 144; const unsigned struct_kernel_stat64_sz = 104; #elif defined(__mips__) - #if SANITIZER_WORDSIZE == 64 - const unsigned struct_kernel_stat_sz = 216; - #else - const unsigned struct_kernel_stat_sz = 144; - #endif + const unsigned struct_kernel_stat_sz = + SANITIZER_ANDROID ? FIRST_32_SECOND_64(104, 128) : + FIRST_32_SECOND_64(144, 216); const unsigned struct_kernel_stat64_sz = 104; -#elif defined(__sparc__) && defined(__arch64__) - const unsigned struct___old_kernel_stat_sz = 0; - const unsigned struct_kernel_stat_sz = 104; - const unsigned struct_kernel_stat64_sz = 144; -#elif defined(__sparc__) && !defined(__arch64__) - const unsigned struct___old_kernel_stat_sz = 0; +#elif defined(__s390__) && !defined(__s390x__) const unsigned struct_kernel_stat_sz = 64; const unsigned struct_kernel_stat64_sz = 104; +#elif defined(__s390x__) + const unsigned struct_kernel_stat_sz = 144; + const unsigned struct_kernel_stat64_sz = 0; #endif struct __sanitizer_perf_event_attr { unsigned type; @@ -109,9 +105,9 @@ namespace __sanitizer { #if SANITIZER_LINUX || SANITIZER_FREEBSD -#if defined(__powerpc64__) +#if defined(__powerpc64__) || defined(__s390__) const unsigned struct___old_kernel_stat_sz = 0; -#elif !defined(__sparc__) +#else const unsigned struct___old_kernel_stat_sz = 32; #endif @@ -196,24 +192,12 @@ namespace __sanitizer { unsigned __seq; u64 __unused1; u64 __unused2; -#elif defined(__mips__) || defined(__aarch64__) +#elif defined(__mips__) || defined(__aarch64__) || defined(__s390x__) unsigned int mode; unsigned short __seq; unsigned short __pad1; unsigned long __unused1; unsigned long __unused2; -#elif defined(__sparc__) -# if defined(__arch64__) - unsigned mode; - unsigned short __pad1; -# else - unsigned short __pad1; - unsigned short mode; - unsigned short __pad2; -# endif - unsigned short __seq; - unsigned long long __unused1; - unsigned long long __unused2; #else unsigned short mode; unsigned short __pad1; @@ -231,26 +215,6 @@ namespace __sanitizer { struct __sanitizer_shmid_ds { __sanitizer_ipc_perm shm_perm; - #if defined(__sparc__) - # if !defined(__arch64__) - u32 __pad1; - # endif - long shm_atime; - # if !defined(__arch64__) - u32 __pad2; - # endif - long shm_dtime; - # if !defined(__arch64__) - u32 __pad3; - # endif - long shm_ctime; - uptr shm_segsz; - int shm_cpid; - int shm_lpid; - unsigned long shm_nattch; - unsigned long __glibc_reserved1; - unsigned long __glibc_reserved2; - #else #ifndef __powerpc__ uptr shm_segsz; #elif !defined(__powerpc64__) @@ -288,7 +252,6 @@ namespace __sanitizer { uptr __unused4; uptr __unused5; #endif -#endif }; #elif SANITIZER_FREEBSD struct __sanitizer_ipc_perm { @@ -555,7 +518,11 @@ namespace __sanitizer { }; #if SANITIZER_ANDROID +# if SANITIZER_MIPS + typedef unsigned long __sanitizer_sigset_t[16/sizeof(unsigned long)]; +# else typedef unsigned long __sanitizer_sigset_t; +# endif #elif SANITIZER_MAC typedef unsigned __sanitizer_sigset_t; #elif SANITIZER_LINUX @@ -581,6 +548,15 @@ namespace __sanitizer { __sanitizer_sigset_t sa_mask; void (*sa_restorer)(); }; +#elif SANITIZER_ANDROID && SANITIZER_MIPS32 // check this before WORDSIZE == 32 + struct __sanitizer_sigaction { + unsigned sa_flags; + union { + void (*sigaction)(int sig, void *siginfo, void *uctx); + void (*handler)(int sig); + }; + __sanitizer_sigset_t sa_mask; + }; #elif SANITIZER_ANDROID && (SANITIZER_WORDSIZE == 32) struct __sanitizer_sigaction { union { @@ -604,21 +580,24 @@ namespace __sanitizer { int sa_flags; __sanitizer_sigset_t sa_mask; #else +#if defined(__s390x__) + int sa_resv; +#else __sanitizer_sigset_t sa_mask; +#endif #ifndef __mips__ -#if defined(__sparc__) - unsigned long sa_flags; -#else int sa_flags; #endif #endif -#endif #if SANITIZER_LINUX void (*sa_restorer)(); #endif #if defined(__mips__) && (SANITIZER_WORDSIZE == 32) int sa_resv[1]; #endif +#if defined(__s390x__) + __sanitizer_sigset_t sa_mask; +#endif }; #endif // !SANITIZER_ANDROID @@ -626,7 +605,7 @@ namespace __sanitizer { typedef __sanitizer_sigset_t __sanitizer_kernel_sigset_t; #elif defined(__mips__) struct __sanitizer_kernel_sigset_t { - u8 sig[16]; + uptr sig[2]; }; #else struct __sanitizer_kernel_sigset_t { @@ -635,6 +614,17 @@ namespace __sanitizer { #endif // Linux system headers define the 'sa_handler' and 'sa_sigaction' macros. +#if SANITIZER_MIPS + struct __sanitizer_kernel_sigaction_t { + unsigned int sa_flags; + union { + void (*handler)(int signo); + void (*sigaction)(int signo, void *info, void *ctx); + }; + __sanitizer_kernel_sigset_t sa_mask; + void (*sa_restorer)(void); + }; +#else struct __sanitizer_kernel_sigaction_t { union { void (*handler)(int signo); @@ -644,6 +634,7 @@ namespace __sanitizer { void (*sa_restorer)(void); __sanitizer_kernel_sigset_t sa_mask; }; +#endif extern uptr sig_ign; extern uptr sig_dfl; @@ -779,7 +770,8 @@ namespace __sanitizer { #if SANITIZER_LINUX && !SANITIZER_ANDROID && \ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \ - defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__)) + defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \ + defined(__s390__)) extern unsigned struct_user_regs_struct_sz; extern unsigned struct_user_fpregs_struct_sz; extern unsigned struct_user_fpxregs_struct_sz; @@ -857,7 +849,7 @@ struct __sanitizer_cookie_io_functions_t { #define IOC_NRBITS 8 #define IOC_TYPEBITS 8 -#if defined(__powerpc__) || defined(__powerpc64__) || defined(__mips__) || defined(__sparc__) +#if defined(__powerpc__) || defined(__powerpc64__) || defined(__mips__) #define IOC_SIZEBITS 13 #define IOC_DIRBITS 3 #define IOC_NONE 1U @@ -887,17 +879,7 @@ struct __sanitizer_cookie_io_functions_t { #define IOC_DIR(nr) (((nr) >> IOC_DIRSHIFT) & IOC_DIRMASK) #define IOC_TYPE(nr) (((nr) >> IOC_TYPESHIFT) & IOC_TYPEMASK) #define IOC_NR(nr) (((nr) >> IOC_NRSHIFT) & IOC_NRMASK) - -#if defined(__sparc__) -// In sparc the 14 bits SIZE field overlaps with the -// least significant bit of DIR, so either IOC_READ or -// IOC_WRITE shall be 1 in order to get a non-zero SIZE. -# define IOC_SIZE(nr) \ - ((((((nr) >> 29) & 0x7) & (4U|2U)) == 0)? \ - 0 : (((nr) >> 16) & 0x3fff)) -#else #define IOC_SIZE(nr) (((nr) >> IOC_SIZESHIFT) & IOC_SIZEMASK) -#endif extern unsigned struct_ifreq_sz; extern unsigned struct_termios_sz; diff --git a/libsanitizer/sanitizer_common/sanitizer_posix.cc b/libsanitizer/sanitizer_common/sanitizer_posix.cc index ed44633bc182..d10213d917f9 100644 --- a/libsanitizer/sanitizer_common/sanitizer_posix.cc +++ b/libsanitizer/sanitizer_common/sanitizer_posix.cc @@ -87,7 +87,11 @@ static uptr GetKernelAreaSize() { uptr GetMaxVirtualAddress() { #if SANITIZER_WORDSIZE == 64 -# if defined(__powerpc64__) || defined(__aarch64__) +# if defined(__aarch64__) && SANITIZER_IOS && !SANITIZER_IOSSIM + // Ideally, we would derive the upper bound from MACH_VM_MAX_ADDRESS. The + // upper bound can change depending on the device. + return 0x200000000 - 1; +# elif defined(__powerpc64__) || defined(__aarch64__) // On PowerPC64 we have two different address space layouts: 44- and 46-bit. // We somehow need to figure out which one we are using now and choose // one of 0x00000fffffffffffUL and 0x00003fffffffffffUL. @@ -98,26 +102,32 @@ uptr GetMaxVirtualAddress() { return (1ULL << (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1)) - 1; # elif defined(__mips64) return (1ULL << 40) - 1; // 0x000000ffffffffffUL; +# elif defined(__s390x__) + return (1ULL << 53) - 1; // 0x001fffffffffffffUL; # else return (1ULL << 47) - 1; // 0x00007fffffffffffUL; # endif #else // SANITIZER_WORDSIZE == 32 +# if defined(__s390__) + return (1ULL << 31) - 1; // 0x7fffffff; +# else uptr res = (1ULL << 32) - 1; // 0xffffffff; if (!common_flags()->full_address_space) res -= GetKernelAreaSize(); CHECK_LT(reinterpret_cast<uptr>(&res), res); return res; +# endif #endif // SANITIZER_WORDSIZE } -void *MmapOrDie(uptr size, const char *mem_type) { +void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) { size = RoundUpTo(size, GetPageSizeCached()); uptr res = internal_mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); int reserrno; if (internal_iserror(res, &reserrno)) - ReportMmapFailureAndDie(size, mem_type, "allocate", reserrno); + ReportMmapFailureAndDie(size, mem_type, "allocate", reserrno, raw_report); IncreaseTotalMmap(size); return (void *)res; } @@ -133,6 +143,26 @@ void UnmapOrDie(void *addr, uptr size) { DecreaseTotalMmap(size); } +// We want to map a chunk of address space aligned to 'alignment'. +// We do it by maping a bit more and then unmaping redundant pieces. +// We probably can do it with fewer syscalls in some OS-dependent way. +void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type) { + CHECK(IsPowerOfTwo(size)); + CHECK(IsPowerOfTwo(alignment)); + uptr map_size = size + alignment; + uptr map_res = (uptr)MmapOrDie(map_size, mem_type); + uptr map_end = map_res + map_size; + uptr res = map_res; + if (res & (alignment - 1)) // Not aligned. + res = (map_res + alignment) & ~(alignment - 1); + uptr end = res + size; + if (res != map_res) + UnmapOrDie((void*)map_res, res - map_res); + if (end != map_end) + UnmapOrDie((void*)end, map_end - end); + return (void*)res; +} + void *MmapNoReserveOrDie(uptr size, const char *mem_type) { uptr PageSize = GetPageSizeCached(); uptr p = internal_mmap(nullptr, @@ -169,6 +199,10 @@ bool MprotectNoAccess(uptr addr, uptr size) { return 0 == internal_mprotect((void*)addr, size, PROT_NONE); } +bool MprotectReadOnly(uptr addr, uptr size) { + return 0 == internal_mprotect((void *)addr, size, PROT_READ); +} + fd_t OpenFile(const char *filename, FileAccessMode mode, error_t *errno_p) { int flags; switch (mode) { @@ -313,26 +347,13 @@ bool GetCodeRangeForFile(const char *module, uptr *start, uptr *end) { } SignalContext SignalContext::Create(void *siginfo, void *context) { - uptr addr = (uptr)((siginfo_t*)siginfo)->si_addr; + auto si = (siginfo_t *)siginfo; + uptr addr = (uptr)si->si_addr; uptr pc, sp, bp; GetPcSpBp(context, &pc, &sp, &bp); - return SignalContext(context, addr, pc, sp, bp); -} - -// This function check is the built VMA matches the runtime one for -// architectures with multiple VMA size. -void CheckVMASize() { -#ifdef __aarch64__ - static const unsigned kBuiltVMA = SANITIZER_AARCH64_VMA; - unsigned maxRuntimeVMA = - (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1); - if (kBuiltVMA != maxRuntimeVMA) { - Printf("WARNING: %s runtime VMA is not the one built for.\n", - SanitizerToolName); - Printf("\tBuilt VMA: %u bits\n", kBuiltVMA); - Printf("\tRuntime VMA: %u bits\n", maxRuntimeVMA); - } -#endif + WriteFlag write_flag = GetWriteFlag(context); + bool is_memory_access = si->si_signo == SIGSEGV; + return SignalContext(context, addr, pc, sp, bp, is_memory_access, write_flag); } } // namespace __sanitizer diff --git a/libsanitizer/sanitizer_common/sanitizer_posix.h b/libsanitizer/sanitizer_common/sanitizer_posix.h index 8dd259e32baf..68b34babdeb8 100644 --- a/libsanitizer/sanitizer_common/sanitizer_posix.h +++ b/libsanitizer/sanitizer_common/sanitizer_posix.h @@ -14,6 +14,7 @@ // ----------- ATTENTION ------------- // This header should NOT include any other headers from sanitizer runtime. #include "sanitizer_internal_defs.h" +#include "sanitizer_platform_limits_posix.h" #if !SANITIZER_POSIX // Make it hard to accidentally use any of functions declared in this file: @@ -52,6 +53,7 @@ uptr internal_ptrace(int request, int pid, void *addr, void *data); uptr internal_waitpid(int pid, int *status, int options); int internal_fork(); +int internal_forkpty(int *amaster); // These functions call appropriate pthread_ functions directly, bypassing // the interceptor. They are weak and may not be present in some tools. @@ -74,8 +76,15 @@ int real_pthread_join(void *th, void **ret); int my_pthread_attr_getstack(void *attr, void **addr, uptr *size); +// A routine named real_sigaction() must be implemented by each sanitizer in +// order for internal_sigaction() to bypass interceptors. int internal_sigaction(int signum, const void *act, void *oldact); +void internal_sigfillset(__sanitizer_sigset_t *set); +void internal_sigemptyset(__sanitizer_sigset_t *set); +bool internal_sigismember(__sanitizer_sigset_t *set, int signum); +uptr internal_execve(const char *filename, char *const argv[], + char *const envp[]); } // namespace __sanitizer #endif // SANITIZER_POSIX_H diff --git a/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cc index 4b7273b4cc03..335aad1660e5 100644 --- a/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cc +++ b/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cc @@ -32,6 +32,7 @@ #include <sys/stat.h> #include <sys/time.h> #include <sys/types.h> +#include <sys/wait.h> #include <unistd.h> #if SANITIZER_FREEBSD @@ -41,6 +42,8 @@ #define MAP_NORESERVE 0 #endif +typedef void (*sa_sigaction_t)(int, siginfo_t *, void *); + namespace __sanitizer { u32 GetUid() { @@ -51,7 +54,7 @@ uptr GetThreadSelf() { return (uptr)pthread_self(); } -void FlushUnneededShadowMemory(uptr addr, uptr size) { +void ReleaseMemoryToOS(uptr addr, uptr size) { madvise((void*)addr, size, MADV_DONTNEED); } @@ -95,6 +98,10 @@ bool StackSizeIsUnlimited() { return (stack_size == RLIM_INFINITY); } +uptr GetStackSizeLimitInBytes() { + return (uptr)getlim(RLIMIT_STACK); +} + void SetStackSizeLimitInBytes(uptr limit) { setlim(RLIMIT_STACK, (rlim_t)limit); CHECK(!StackSizeIsUnlimited()); @@ -119,11 +126,21 @@ void SleepForMillis(int millis) { } void Abort() { +#if !SANITIZER_GO + // If we are handling SIGABRT, unhandle it first. + if (IsHandledDeadlySignal(SIGABRT)) { + struct sigaction sigact; + internal_memset(&sigact, 0, sizeof(sigact)); + sigact.sa_sigaction = (sa_sigaction_t)SIG_DFL; + internal_sigaction(SIGABRT, &sigact, nullptr); + } +#endif + abort(); } int Atexit(void (*function)(void)) { -#ifndef SANITIZER_GO +#if !SANITIZER_GO return atexit(function); #else return 0; @@ -134,7 +151,7 @@ bool SupportsColoredOutput(fd_t fd) { return isatty(fd) != 0; } -#ifndef SANITIZER_GO +#if !SANITIZER_GO // TODO(glider): different tools may require different altstack size. static const uptr kAltStackSize = SIGSTKSZ * 4; // SIGSTKSZ is not enough. @@ -163,10 +180,9 @@ void UnsetAlternateSignalStack() { UnmapOrDie(oldstack.ss_sp, oldstack.ss_size); } -typedef void (*sa_sigaction_t)(int, siginfo_t *, void *); static void MaybeInstallSigaction(int signum, SignalHandlerType handler) { - if (!IsDeadlySignal(signum)) + if (!IsHandledDeadlySignal(signum)) return; struct sigaction sigact; internal_memset(&sigact, 0, sizeof(sigact)); @@ -188,6 +204,7 @@ void InstallDeadlySignalHandlers(SignalHandlerType handler) { MaybeInstallSigaction(SIGBUS, handler); MaybeInstallSigaction(SIGABRT, handler); MaybeInstallSigaction(SIGFPE, handler); + MaybeInstallSigaction(SIGILL, handler); } #endif // SANITIZER_GO @@ -266,7 +283,7 @@ void *MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) { return (void *)p; } -void *MmapNoAccess(uptr fixed_addr, uptr size, const char *name) { +void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) { int fd = name ? GetNamedMappingFd(name, size) : -1; unsigned flags = MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE; if (fd == -1) flags |= MAP_ANON; @@ -275,6 +292,11 @@ void *MmapNoAccess(uptr fixed_addr, uptr size, const char *name) { 0); } +void *MmapNoAccess(uptr size) { + unsigned flags = MAP_PRIVATE | MAP_ANON | MAP_NORESERVE; + return (void *)internal_mmap(nullptr, size, PROT_NONE, flags, -1, 0); +} + // This function is defined elsewhere if we intercepted pthread_attr_getstack. extern "C" { SANITIZER_WEAK_ATTRIBUTE int @@ -317,6 +339,79 @@ void AdjustStackSize(void *attr_) { } #endif // !SANITIZER_GO +pid_t StartSubprocess(const char *program, const char *const argv[], + fd_t stdin_fd, fd_t stdout_fd, fd_t stderr_fd) { + auto file_closer = at_scope_exit([&] { + if (stdin_fd != kInvalidFd) { + internal_close(stdin_fd); + } + if (stdout_fd != kInvalidFd) { + internal_close(stdout_fd); + } + if (stderr_fd != kInvalidFd) { + internal_close(stderr_fd); + } + }); + + int pid = internal_fork(); + + if (pid < 0) { + int rverrno; + if (internal_iserror(pid, &rverrno)) { + Report("WARNING: failed to fork (errno %d)\n", rverrno); + } + return pid; + } + + if (pid == 0) { + // Child subprocess + if (stdin_fd != kInvalidFd) { + internal_close(STDIN_FILENO); + internal_dup2(stdin_fd, STDIN_FILENO); + internal_close(stdin_fd); + } + if (stdout_fd != kInvalidFd) { + internal_close(STDOUT_FILENO); + internal_dup2(stdout_fd, STDOUT_FILENO); + internal_close(stdout_fd); + } + if (stderr_fd != kInvalidFd) { + internal_close(STDERR_FILENO); + internal_dup2(stderr_fd, STDERR_FILENO); + internal_close(stderr_fd); + } + + for (int fd = sysconf(_SC_OPEN_MAX); fd > 2; fd--) internal_close(fd); + + execv(program, const_cast<char **>(&argv[0])); + internal__exit(1); + } + + return pid; +} + +bool IsProcessRunning(pid_t pid) { + int process_status; + uptr waitpid_status = internal_waitpid(pid, &process_status, WNOHANG); + int local_errno; + if (internal_iserror(waitpid_status, &local_errno)) { + VReport(1, "Waiting on the process failed (errno %d).\n", local_errno); + return false; + } + return waitpid_status == 0; +} + +int WaitForProcess(pid_t pid) { + int process_status; + uptr waitpid_status = internal_waitpid(pid, &process_status, 0); + int local_errno; + if (internal_iserror(waitpid_status, &local_errno)) { + VReport(1, "Waiting on the process failed (errno %d).\n", local_errno); + return -1; + } + return process_status; +} + } // namespace __sanitizer #endif // SANITIZER_POSIX diff --git a/libsanitizer/sanitizer_common/sanitizer_printf.cc b/libsanitizer/sanitizer_common/sanitizer_printf.cc index 6688610bf0f4..c11113da244e 100644 --- a/libsanitizer/sanitizer_common/sanitizer_printf.cc +++ b/libsanitizer/sanitizer_common/sanitizer_printf.cc @@ -211,7 +211,7 @@ SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void OnPrint(const char *str) { (void)str; } -#elif defined(SANITIZER_GO) && defined(TSAN_EXTERNAL_HOOKS) +#elif SANITIZER_GO && defined(TSAN_EXTERNAL_HOOKS) void OnPrint(const char *str); #else void OnPrint(const char *str) { @@ -276,9 +276,12 @@ static void SharedPrintfCode(bool append_pid, const char *format, # undef CHECK_NEEDED_LENGTH } RawWrite(buffer); - if (common_flags()->log_to_syslog) - WriteToSyslog(buffer); + + // Remove color sequences from the message. + RemoveANSIEscapeSequencesFromString(buffer); CallPrintfAndReportCallback(buffer); + LogMessageOnPrintf(buffer); + // If we had mapped any memory, clean up. if (buffer != local_buffer) UnmapOrDie((void *)buffer, buffer_size); diff --git a/libsanitizer/sanitizer_common/sanitizer_procmaps.h b/libsanitizer/sanitizer_common/sanitizer_procmaps.h index 7477abf30b67..0183a0941119 100644 --- a/libsanitizer/sanitizer_common/sanitizer_procmaps.h +++ b/libsanitizer/sanitizer_common/sanitizer_procmaps.h @@ -41,9 +41,8 @@ class MemoryMappingLayout { // instead of aborting. static void CacheMemoryMappings(); - // Stores the list of mapped objects into an array. - uptr DumpListOfModules(LoadedModule *modules, uptr max_modules, - string_predicate_t filter); + // Adds all mapped objects into a vector. + void DumpListOfModules(InternalMmapVector<LoadedModule> *modules); // Memory protection masks. static const uptr kProtectionRead = 1; diff --git a/libsanitizer/sanitizer_common/sanitizer_procmaps_common.cc b/libsanitizer/sanitizer_common/sanitizer_procmaps_common.cc index 1cf3f8510fb6..c725c2e7b664 100644 --- a/libsanitizer/sanitizer_common/sanitizer_procmaps_common.cc +++ b/libsanitizer/sanitizer_common/sanitizer_procmaps_common.cc @@ -114,22 +114,17 @@ void MemoryMappingLayout::LoadFromCache() { } } -uptr MemoryMappingLayout::DumpListOfModules(LoadedModule *modules, - uptr max_modules, - string_predicate_t filter) { +void MemoryMappingLayout::DumpListOfModules( + InternalMmapVector<LoadedModule> *modules) { Reset(); uptr cur_beg, cur_end, cur_offset, prot; InternalScopedString module_name(kMaxPathLength); - uptr n_modules = 0; - for (uptr i = 0; n_modules < max_modules && - Next(&cur_beg, &cur_end, &cur_offset, module_name.data(), - module_name.size(), &prot); + for (uptr i = 0; Next(&cur_beg, &cur_end, &cur_offset, module_name.data(), + module_name.size(), &prot); i++) { const char *cur_name = module_name.data(); if (cur_name[0] == '\0') continue; - if (filter && !filter(cur_name)) - continue; // Don't subtract 'cur_beg' from the first entry: // * If a binary is compiled w/o -pie, then the first entry in // process maps is likely the binary itself (all dynamic libs @@ -142,12 +137,11 @@ uptr MemoryMappingLayout::DumpListOfModules(LoadedModule *modules, // shadow memory of the tool), so the module can't be the // first entry. uptr base_address = (i ? cur_beg : 0) - cur_offset; - LoadedModule *cur_module = &modules[n_modules]; - cur_module->set(cur_name, base_address); - cur_module->addAddressRange(cur_beg, cur_end, prot & kProtectionExecute); - n_modules++; + LoadedModule cur_module; + cur_module.set(cur_name, base_address); + cur_module.addAddressRange(cur_beg, cur_end, prot & kProtectionExecute); + modules->push_back(cur_module); } - return n_modules; } void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) { diff --git a/libsanitizer/sanitizer_common/sanitizer_procmaps_linux.cc b/libsanitizer/sanitizer_common/sanitizer_procmaps_linux.cc index e77e33fb2d77..10918e543988 100644 --- a/libsanitizer/sanitizer_common/sanitizer_procmaps_linux.cc +++ b/libsanitizer/sanitizer_common/sanitizer_procmaps_linux.cc @@ -65,7 +65,7 @@ bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset, while (IsDecimal(*current_)) current_++; // Qemu may lack the trailing space. - // http://code.google.com/p/address-sanitizer/issues/detail?id=160 + // https://github.com/google/sanitizers/issues/160 // CHECK_EQ(*current_++, ' '); // Skip spaces. while (current_ < next_line && *current_ == ' ') diff --git a/libsanitizer/sanitizer_common/sanitizer_procmaps_mac.cc b/libsanitizer/sanitizer_common/sanitizer_procmaps_mac.cc index 393243b7c281..81829a7c2842 100644 --- a/libsanitizer/sanitizer_common/sanitizer_procmaps_mac.cc +++ b/libsanitizer/sanitizer_common/sanitizer_procmaps_mac.cc @@ -63,7 +63,7 @@ void MemoryMappingLayout::LoadFromCache() { } // Next and NextSegmentLoad were inspired by base/sysinfo.cc in -// Google Perftools, http://code.google.com/p/google-perftools. +// Google Perftools, https://github.com/gperftools/gperftools. // NextSegmentLoad scans the current image for the next segment load command // and returns the start and end addresses and file offset of the corresponding @@ -153,34 +153,28 @@ bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset, return false; } -uptr MemoryMappingLayout::DumpListOfModules(LoadedModule *modules, - uptr max_modules, - string_predicate_t filter) { +void MemoryMappingLayout::DumpListOfModules( + InternalMmapVector<LoadedModule> *modules) { Reset(); uptr cur_beg, cur_end, prot; InternalScopedString module_name(kMaxPathLength); - uptr n_modules = 0; - for (uptr i = 0; n_modules < max_modules && - Next(&cur_beg, &cur_end, 0, module_name.data(), - module_name.size(), &prot); + for (uptr i = 0; Next(&cur_beg, &cur_end, 0, module_name.data(), + module_name.size(), &prot); i++) { const char *cur_name = module_name.data(); if (cur_name[0] == '\0') continue; - if (filter && !filter(cur_name)) - continue; LoadedModule *cur_module = nullptr; - if (n_modules > 0 && - 0 == internal_strcmp(cur_name, modules[n_modules - 1].full_name())) { - cur_module = &modules[n_modules - 1]; + if (!modules->empty() && + 0 == internal_strcmp(cur_name, modules->back().full_name())) { + cur_module = &modules->back(); } else { - cur_module = &modules[n_modules]; + modules->push_back(LoadedModule()); + cur_module = &modules->back(); cur_module->set(cur_name, cur_beg); - n_modules++; } cur_module->addAddressRange(cur_beg, cur_end, prot & kProtectionExecute); } - return n_modules; } } // namespace __sanitizer diff --git a/libsanitizer/sanitizer_common/sanitizer_quarantine.h b/libsanitizer/sanitizer_common/sanitizer_quarantine.h index a635871be454..9e9268f2a5d2 100644 --- a/libsanitizer/sanitizer_common/sanitizer_quarantine.h +++ b/libsanitizer/sanitizer_common/sanitizer_quarantine.h @@ -99,10 +99,12 @@ class Quarantine { void NOINLINE DoRecycle(Cache *c, Callback cb) { while (QuarantineBatch *b = c->DequeueBatch()) { const uptr kPrefetch = 16; + CHECK(kPrefetch <= ARRAY_SIZE(b->batch)); for (uptr i = 0; i < kPrefetch; i++) PREFETCH(b->batch[i]); - for (uptr i = 0; i < b->count; i++) { - PREFETCH(b->batch[i + kPrefetch]); + for (uptr i = 0, count = b->count; i < count; i++) { + if (i + kPrefetch < count) + PREFETCH(b->batch[i + kPrefetch]); cb.Recycle((Node*)b->batch[i]); } cb.Deallocate(b); diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace.cc b/libsanitizer/sanitizer_common/sanitizer_stacktrace.cc index 796d472a1eba..531f256c0489 100644 --- a/libsanitizer/sanitizer_common/sanitizer_stacktrace.cc +++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace.cc @@ -38,11 +38,6 @@ void BufferedStackTrace::Init(const uptr *pcs, uptr cnt, uptr extra_top_pc) { top_frame_bp = 0; } -// Check if given pointer points into allocated stack area. -static inline bool IsValidFrame(uptr frame, uptr stack_top, uptr stack_bottom) { - return frame > stack_bottom && frame < stack_top - 2 * sizeof (uhwptr); -} - // In GCC on ARM bp points to saved lr, not fp, so we should check the next // cell in stack to be a saved frame pointer. GetCanonicFrame returns the // pointer to saved frame pointer in any case. @@ -60,8 +55,8 @@ static inline uhwptr *GetCanonicFrame(uptr bp, // Nope, this does not look right either. This means the frame after next does // not have a valid frame pointer, but we can still extract the caller PC. // Unfortunately, there is no way to decide between GCC and LLVM frame - // layouts. Assume GCC. - return bp_prev - 1; + // layouts. Assume LLVM. + return bp_prev; #else return (uhwptr*)bp; #endif @@ -69,6 +64,7 @@ static inline uhwptr *GetCanonicFrame(uptr bp, void BufferedStackTrace::FastUnwindStack(uptr pc, uptr bp, uptr stack_top, uptr stack_bottom, u32 max_depth) { + const uptr kPageSize = GetPageSizeCached(); CHECK_GE(max_depth, 2); trace_buffer[0] = pc; size = 1; @@ -90,9 +86,16 @@ void BufferedStackTrace::FastUnwindStack(uptr pc, uptr bp, uptr stack_top, !IsAligned((uptr)caller_frame, sizeof(uhwptr))) break; uhwptr pc1 = caller_frame[2]; +#elif defined(__s390__) + uhwptr pc1 = frame[14]; #else uhwptr pc1 = frame[1]; #endif + // Let's assume that any pointer in the 0th page (i.e. <0x1000 on i386 and + // x86_64) is invalid and stop unwinding here. If we're adding support for + // a platform where this isn't true, we need to reconsider this check. + if (pc1 < kPageSize) + break; if (pc1 != pc) { trace_buffer[size++] = (uptr) pc1; } @@ -116,7 +119,7 @@ void BufferedStackTrace::PopStackFrames(uptr count) { uptr BufferedStackTrace::LocatePcInTrace(uptr pc) { // Use threshold to find PC in stack trace, as PC we want to unwind from may // slightly differ from return address in the actual unwinded stack trace. - const int kPcThreshold = 304; + const int kPcThreshold = 350; for (uptr i = 0; i < size; ++i) { if (MatchPc(pc, trace[i], kPcThreshold)) return i; diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace.h b/libsanitizer/sanitizer_common/sanitizer_stacktrace.h index 7f22455ac9e3..c59dbd558838 100644 --- a/libsanitizer/sanitizer_common/sanitizer_stacktrace.h +++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace.h @@ -27,7 +27,7 @@ static const u32 kStackTraceMax = 256; // Fast unwind is the only option on Mac for now; we will need to // revisit this macro when slow unwind works on Mac, see -// https://code.google.com/p/address-sanitizer/issues/detail?id=137 +// https://github.com/google/sanitizers/issues/137 #if SANITIZER_MAC # define SANITIZER_CAN_SLOW_UNWIND 0 #else @@ -108,6 +108,11 @@ struct BufferedStackTrace : public StackTrace { void operator=(const BufferedStackTrace &); }; +// Check if given pointer points into allocated stack area. +static inline bool IsValidFrame(uptr frame, uptr stack_top, uptr stack_bottom) { + return frame > stack_bottom && frame < stack_top - 2 * sizeof (uhwptr); +} + } // namespace __sanitizer // Use this macro if you want to print stack trace with the caller diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_stacktrace_libcdep.cc index addf44f73279..ac3ee3a019c2 100644 --- a/libsanitizer/sanitizer_common/sanitizer_stacktrace_libcdep.cc +++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace_libcdep.cc @@ -23,6 +23,8 @@ void StackTrace::Print() const { return; } InternalScopedString frame_desc(GetPageSizeCached() * 2); + InternalScopedString dedup_token(GetPageSizeCached()); + int dedup_frames = common_flags()->dedup_token_length; uptr frame_num = 0; for (uptr i = 0; i < size && trace[i]; i++) { // PCs in stack traces are actually the return addresses, that is, @@ -36,11 +38,18 @@ void StackTrace::Print() const { cur->info, common_flags()->symbolize_vs_style, common_flags()->strip_path_prefix); Printf("%s\n", frame_desc.data()); + if (dedup_frames-- > 0) { + if (dedup_token.length()) + dedup_token.append("--"); + dedup_token.append(cur->info.function); + } } frames->ClearAll(); } // Always print a trailing empty line after stack trace. Printf("\n"); + if (dedup_token.length()) + Printf("DEDUP_TOKEN: %s\n", dedup_token.data()); } void BufferedStackTrace::Unwind(u32 max_depth, uptr pc, uptr bp, void *context, @@ -72,3 +81,38 @@ void BufferedStackTrace::Unwind(u32 max_depth, uptr pc, uptr bp, void *context, } } // namespace __sanitizer +using namespace __sanitizer; + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_symbolize_pc(uptr pc, const char *fmt, char *out_buf, + uptr out_buf_size) { + if (!out_buf_size) return; + pc = StackTrace::GetPreviousInstructionPc(pc); + SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc); + if (!frame) { + internal_strncpy(out_buf, "<can't symbolize>", out_buf_size); + out_buf[out_buf_size - 1] = 0; + return; + } + InternalScopedString frame_desc(GetPageSizeCached()); + RenderFrame(&frame_desc, fmt, 0, frame->info, + common_flags()->symbolize_vs_style, + common_flags()->strip_path_prefix); + internal_strncpy(out_buf, frame_desc.data(), out_buf_size); + out_buf[out_buf_size - 1] = 0; +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_symbolize_global(uptr data_addr, const char *fmt, + char *out_buf, uptr out_buf_size) { + if (!out_buf_size) return; + out_buf[0] = 0; + DataInfo DI; + if (!Symbolizer::GetOrInit()->SymbolizeData(data_addr, &DI)) return; + InternalScopedString data_desc(GetPageSizeCached()); + RenderData(&data_desc, fmt, &DI, common_flags()->strip_path_prefix); + internal_strncpy(out_buf, data_desc.data(), out_buf_size); + out_buf[out_buf_size - 1] = 0; +} +} // extern "C" diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.cc b/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.cc index bcc9de78ec15..de78c7ac8d1d 100644 --- a/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.cc +++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.cc @@ -114,6 +114,35 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, } } +void RenderData(InternalScopedString *buffer, const char *format, + const DataInfo *DI, const char *strip_path_prefix) { + for (const char *p = format; *p != '\0'; p++) { + if (*p != '%') { + buffer->append("%c", *p); + continue; + } + p++; + switch (*p) { + case '%': + buffer->append("%%"); + break; + case 's': + buffer->append("%s", StripPathPrefix(DI->file, strip_path_prefix)); + break; + case 'l': + buffer->append("%d", DI->line); + break; + case 'g': + buffer->append("%s", DI->name); + break; + default: + Report("Unsupported specifier in stack frame format: %c (0x%zx)!\n", *p, + *p); + Die(); + } + } +} + void RenderSourceLocation(InternalScopedString *buffer, const char *file, int line, int column, bool vs_style, const char *strip_path_prefix) { diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.h b/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.h index a553568ea6ff..6726f141f225 100644 --- a/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.h +++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.h @@ -57,6 +57,13 @@ void RenderSourceLocation(InternalScopedString *buffer, const char *file, void RenderModuleLocation(InternalScopedString *buffer, const char *module, uptr offset, const char *strip_path_prefix); +// Same as RenderFrame, but for data section (global variables). +// Accepts %s, %l from above. +// Also accepts: +// %g - name of the global variable. +void RenderData(InternalScopedString *buffer, const char *format, + const DataInfo *DI, const char *strip_path_prefix = ""); + } // namespace __sanitizer #endif // SANITIZER_STACKTRACE_PRINTER_H diff --git a/libsanitizer/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc index c919e4f6e979..891386dc0ba7 100644 --- a/libsanitizer/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc +++ b/libsanitizer/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc @@ -13,7 +13,8 @@ #include "sanitizer_platform.h" #if SANITIZER_LINUX && (defined(__x86_64__) || defined(__mips__) || \ - defined(__aarch64__)) + defined(__aarch64__) || defined(__powerpc64__) || \ + defined(__s390__)) #include "sanitizer_stoptheworld.h" @@ -36,6 +37,9 @@ # include <asm/ptrace.h> # endif # include <sys/user.h> // for user_regs_struct +# if SANITIZER_ANDROID && SANITIZER_MIPS +# include <asm/reg.h> // for mips SP register in sys/user.h +# endif #endif #include <sys/wait.h> // for signal-related stuff @@ -73,10 +77,10 @@ // thread-local variables used by libc will be shared between the tracer task // and the thread which spawned it. -COMPILER_CHECK(sizeof(SuspendedThreadID) == sizeof(pid_t)); - namespace __sanitizer { +COMPILER_CHECK(sizeof(SuspendedThreadID) == sizeof(pid_t)); + // Structure for passing arguments into the tracer thread. struct TracerThreadArgument { StopTheWorldCallback callback; @@ -184,6 +188,7 @@ void ThreadSuspender::KillAllThreads() { bool ThreadSuspender::SuspendAllThreads() { ThreadLister thread_lister(pid_); bool added_threads; + bool first_iteration = true; do { // Run through the directory entries once. added_threads = false; @@ -193,12 +198,13 @@ bool ThreadSuspender::SuspendAllThreads() { added_threads = true; tid = thread_lister.GetNextTID(); } - if (thread_lister.error()) { + if (thread_lister.error() || (first_iteration && !added_threads)) { // Detach threads and fail. ResumeAllThreads(); return false; } thread_lister.Reset(); + first_iteration = false; } while (added_threads); return true; } @@ -227,8 +233,8 @@ static void TracerThreadDieCallback() { // Signal handler to wake up suspended threads when the tracer thread dies. static void TracerThreadSignalHandler(int signum, void *siginfo, void *uctx) { SignalContext ctx = SignalContext::Create(siginfo, uctx); - VPrintf(1, "Tracer caught signal %d: addr=0x%zx pc=0x%zx sp=0x%zx\n", - signum, ctx.addr, ctx.pc, ctx.sp); + Printf("Tracer caught signal %d: addr=0x%zx pc=0x%zx sp=0x%zx\n", signum, + ctx.addr, ctx.pc, ctx.sp); ThreadSuspender *inst = thread_suspender_instance; if (inst) { if (signum == SIGABRT) @@ -465,13 +471,22 @@ typedef pt_regs regs_struct; #elif defined(__mips__) typedef struct user regs_struct; -#define REG_SP regs[EF_REG29] +# if SANITIZER_ANDROID +# define REG_SP regs[EF_R29] +# else +# define REG_SP regs[EF_REG29] +# endif #elif defined(__aarch64__) typedef struct user_pt_regs regs_struct; #define REG_SP sp #define ARCH_IOVEC_FOR_GETREGSET +#elif defined(__s390__) +typedef _user_regs_struct regs_struct; +#define REG_SP gprs[15] +#define ARCH_IOVEC_FOR_GETREGSET + #else #error "Unsupported architecture" #endif // SANITIZER_ANDROID && defined(__arm__) @@ -509,5 +524,6 @@ uptr SuspendedThreadsList::RegisterCount() { } } // namespace __sanitizer -#endif // SANITIZER_LINUX && (defined(__x86_64__) || defined(__mips__) - // || defined(__aarch64__) +#endif // SANITIZER_LINUX && (defined(__x86_64__) || defined(__mips__) + // || defined(__aarch64__) || defined(__powerpc64__) + // || defined(__s390__) diff --git a/libsanitizer/sanitizer_common/sanitizer_suppressions.cc b/libsanitizer/sanitizer_common/sanitizer_suppressions.cc index 2b6a78155fd2..bfdff59a35cd 100644 --- a/libsanitizer/sanitizer_common/sanitizer_suppressions.cc +++ b/libsanitizer/sanitizer_common/sanitizer_suppressions.cc @@ -125,7 +125,7 @@ void SuppressionContext::Parse(const char *str) { Printf("%s: failed to parse suppressions\n", SanitizerToolName); Die(); } - Suppression s = {}; + Suppression s; s.type = suppression_types_[type]; s.templ = (char*)InternalAlloc(end2 - line + 1); internal_memcpy(s.templ, line, end2 - line); diff --git a/libsanitizer/sanitizer_common/sanitizer_suppressions.h b/libsanitizer/sanitizer_common/sanitizer_suppressions.h index efec926476ba..ed6d7baae846 100644 --- a/libsanitizer/sanitizer_common/sanitizer_suppressions.h +++ b/libsanitizer/sanitizer_common/sanitizer_suppressions.h @@ -18,6 +18,7 @@ namespace __sanitizer { struct Suppression { + Suppression() { internal_memset(this, 0, sizeof(*this)); } const char *type; char *templ; atomic_uint32_t hit_count; @@ -40,7 +41,7 @@ class SuppressionContext { void GetMatched(InternalMmapVector<Suppression *> *matched); private: - static const int kMaxSuppressionTypes = 16; + static const int kMaxSuppressionTypes = 32; const char **const suppression_types_; const int suppression_types_num_; diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer.cc b/libsanitizer/sanitizer_common/sanitizer_symbolizer.cc index 0e58d5fed6e4..3557415aeabd 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer.cc +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer.cc @@ -58,6 +58,7 @@ DataInfo::DataInfo() { void DataInfo::Clear() { InternalFree(module); + InternalFree(file); InternalFree(name); internal_memset(this, 0, sizeof(DataInfo)); } @@ -94,7 +95,7 @@ const char *Symbolizer::ModuleNameOwner::GetOwnedCopy(const char *str) { } Symbolizer::Symbolizer(IntrusiveList<SymbolizerTool> tools) - : module_names_(&mu_), n_modules_(0), modules_fresh_(false), tools_(tools), + : module_names_(&mu_), modules_(), modules_fresh_(false), tools_(tools), start_hook_(0), end_hook_(0) {} Symbolizer::SymbolizerScope::SymbolizerScope(const Symbolizer *sym) diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer.h b/libsanitizer/sanitizer_common/sanitizer_symbolizer.h index 0a443a701156..2b90b42e2ba0 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer.h +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer.h @@ -63,6 +63,8 @@ struct DataInfo { // (de)allocated using sanitizer internal allocator. char *module; uptr module_offset; + char *file; + uptr line; char *name; uptr start; uptr size; @@ -78,6 +80,7 @@ class Symbolizer final { /// Initialize and return platform-specific implementation of symbolizer /// (if it wasn't already initialized). static Symbolizer *GetOrInit(); + static void LateInitialize(); // Returns a list of symbolized frames for a given address (containing // all inlined functions, if necessary). SymbolizedStack *SymbolizePC(uptr address); @@ -111,6 +114,8 @@ class Symbolizer final { void AddHooks(StartSymbolizationHook start_hook, EndSymbolizationHook end_hook); + const LoadedModule *FindModuleForAddress(uptr address); + private: // GetModuleNameAndOffsetForPC has to return a string to the caller. // Since the corresponding module might get unloaded later, we should create @@ -137,9 +142,7 @@ class Symbolizer final { bool FindModuleNameAndOffsetForAddress(uptr address, const char **module_name, uptr *module_offset); - LoadedModule *FindModuleForAddress(uptr address); - LoadedModule modules_[kMaxNumberOfModules]; - uptr n_modules_; + ListOfModules modules_; // If stale, need to reload the modules before looking up addresses. bool modules_fresh_; @@ -155,7 +158,6 @@ class Symbolizer final { // always synchronized. BlockingMutex mu_; - typedef IntrusiveList<SymbolizerTool>::Iterator Iterator; IntrusiveList<SymbolizerTool> tools_; explicit Symbolizer(IntrusiveList<SymbolizerTool> tools); @@ -173,6 +175,10 @@ class Symbolizer final { }; }; +#ifdef SANITIZER_WINDOWS +void InitializeDbgHelpIfNeeded(); +#endif + } // namespace __sanitizer #endif // SANITIZER_SYMBOLIZER_H diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_internal.h b/libsanitizer/sanitizer_common/sanitizer_symbolizer_internal.h index a87964b46361..119cb6884630 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_internal.h +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_internal.h @@ -26,7 +26,7 @@ const char *ExtractUptr(const char *str, const char *delims, uptr *result); const char *ExtractTokenUpToDelimiter(const char *str, const char *delimiter, char **result); -const char *DemangleCXXABI(const char *name); +const char *DemangleSwiftAndCXX(const char *name); // SymbolizerTool is an interface that is implemented by individual "tools" // that can perform symbolication (external llvm-symbolizer, libbacktrace, diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_symbolizer_libcdep.cc index 4264b5e3ddc6..45eb11a4d3b6 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_libcdep.cc +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_libcdep.cc @@ -67,10 +67,9 @@ SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) { return res; // Always fill data about module name and offset. res->info.FillModuleInfo(module_name, module_offset); - for (auto iter = Iterator(&tools_); iter.hasNext();) { - auto *tool = iter.next(); + for (auto &tool : tools_) { SymbolizerScope sym_scope(this); - if (tool->SymbolizePC(addr, res)) { + if (tool.SymbolizePC(addr, res)) { return res; } } @@ -86,10 +85,9 @@ bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) { info->Clear(); info->module = internal_strdup(module_name); info->module_offset = module_offset; - for (auto iter = Iterator(&tools_); iter.hasNext();) { - auto *tool = iter.next(); + for (auto &tool : tools_) { SymbolizerScope sym_scope(this); - if (tool->SymbolizeData(addr, info)) { + if (tool.SymbolizeData(addr, info)) { return true; } } @@ -111,19 +109,17 @@ bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name, void Symbolizer::Flush() { BlockingMutexLock l(&mu_); - for (auto iter = Iterator(&tools_); iter.hasNext();) { - auto *tool = iter.next(); + for (auto &tool : tools_) { SymbolizerScope sym_scope(this); - tool->Flush(); + tool.Flush(); } } const char *Symbolizer::Demangle(const char *name) { BlockingMutexLock l(&mu_); - for (auto iter = Iterator(&tools_); iter.hasNext();) { - auto *tool = iter.next(); + for (auto &tool : tools_) { SymbolizerScope sym_scope(this); - if (const char *demangled = tool->Demangle(name)) + if (const char *demangled = tool.Demangle(name)) return demangled; } return PlatformDemangle(name); @@ -137,27 +133,23 @@ void Symbolizer::PrepareForSandboxing() { bool Symbolizer::FindModuleNameAndOffsetForAddress(uptr address, const char **module_name, uptr *module_offset) { - LoadedModule *module = FindModuleForAddress(address); - if (module == 0) + const LoadedModule *module = FindModuleForAddress(address); + if (module == nullptr) return false; *module_name = module->full_name(); *module_offset = address - module->base_address(); return true; } -LoadedModule *Symbolizer::FindModuleForAddress(uptr address) { +const LoadedModule *Symbolizer::FindModuleForAddress(uptr address) { bool modules_were_reloaded = false; if (!modules_fresh_) { - for (uptr i = 0; i < n_modules_; i++) - modules_[i].clear(); - n_modules_ = - GetListOfModules(modules_, kMaxNumberOfModules, /* filter */ nullptr); - CHECK_GT(n_modules_, 0); - CHECK_LT(n_modules_, kMaxNumberOfModules); + modules_.init(); + RAW_CHECK(modules_.size() > 0); modules_fresh_ = true; modules_were_reloaded = true; } - for (uptr i = 0; i < n_modules_; i++) { + for (uptr i = 0; i < modules_.size(); i++) { if (modules_[i].containsAddress(address)) { return &modules_[i]; } @@ -211,10 +203,18 @@ class LLVMSymbolizerProcess : public SymbolizerProcess { const char* const kSymbolizerArch = "--default-arch=x86_64"; #elif defined(__i386__) const char* const kSymbolizerArch = "--default-arch=i386"; -#elif defined(__powerpc64__) && defined(__BIG_ENDIAN__) +#elif defined(__aarch64__) + const char* const kSymbolizerArch = "--default-arch=arm64"; +#elif defined(__arm__) + const char* const kSymbolizerArch = "--default-arch=arm"; +#elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ const char* const kSymbolizerArch = "--default-arch=powerpc64"; -#elif defined(__powerpc64__) && defined(__LITTLE_ENDIAN__) +#elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ const char* const kSymbolizerArch = "--default-arch=powerpc64le"; +#elif defined(__s390x__) + const char* const kSymbolizerArch = "--default-arch=s390x"; +#elif defined(__s390__) + const char* const kSymbolizerArch = "--default-arch=s390"; #else const char* const kSymbolizerArch = "--default-arch=unknown"; #endif diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_mac.cc b/libsanitizer/sanitizer_common/sanitizer_symbolizer_mac.cc index e65976c18d07..249ccdf83773 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_mac.cc +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_mac.cc @@ -17,8 +17,6 @@ #include "sanitizer_mac.h" #include "sanitizer_symbolizer_mac.h" -namespace __sanitizer { - #include <dlfcn.h> #include <errno.h> #include <stdlib.h> @@ -26,11 +24,14 @@ namespace __sanitizer { #include <unistd.h> #include <util.h> +namespace __sanitizer { + bool DlAddrSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) { Dl_info info; int result = dladdr((const void *)addr, &info); if (!result) return false; - const char *demangled = DemangleCXXABI(info.dli_sname); + const char *demangled = DemangleSwiftAndCXX(info.dli_sname); + if (!demangled) return false; stack->info.function = internal_strdup(demangled); return true; } @@ -39,7 +40,7 @@ bool DlAddrSymbolizer::SymbolizeData(uptr addr, DataInfo *datainfo) { Dl_info info; int result = dladdr((const void *)addr, &info); if (!result) return false; - const char *demangled = DemangleCXXABI(info.dli_sname); + const char *demangled = DemangleSwiftAndCXX(info.dli_sname); datainfo->name = internal_strdup(demangled); datainfo->start = (uptr)info.dli_saddr; return true; @@ -77,23 +78,6 @@ class AtosSymbolizerProcess : public SymbolizerProcess { char pid_str_[16]; }; -static const char *kAtosErrorMessages[] = { - "atos cannot examine process", - "unable to get permission to examine process", - "An admin user name and password is required", - "could not load inserted library", - "architecture mismatch between analysis process", -}; - -static bool IsAtosErrorMessage(const char *str) { - for (uptr i = 0; i < ARRAY_SIZE(kAtosErrorMessages); i++) { - if (internal_strstr(str, kAtosErrorMessages[i])) { - return true; - } - } - return false; -} - static bool ParseCommandOutput(const char *str, uptr addr, char **out_name, char **out_module, char **out_file, uptr *line, uptr *start_address) { @@ -110,15 +94,15 @@ static bool ParseCommandOutput(const char *str, uptr addr, char **out_name, // 0xdeadbeef (in library.dylib) // 0xdeadbeef - if (IsAtosErrorMessage(trim)) { - Report("atos returned an error: %s\n", trim); + const char *rest = trim; + char *symbol_name; + rest = ExtractTokenUpToDelimiter(rest, " (in ", &symbol_name); + if (rest[0] == '\0') { + InternalFree(symbol_name); InternalFree(trim); return false; } - const char *rest = trim; - char *symbol_name; - rest = ExtractTokenUpToDelimiter(rest, " (in ", &symbol_name); if (internal_strncmp(symbol_name, "0x", 2) != 0) *out_name = symbol_name; else @@ -149,6 +133,7 @@ AtosSymbolizer::AtosSymbolizer(const char *path, LowLevelAllocator *allocator) bool AtosSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) { if (!process_) return false; + if (addr == 0) return false; char command[32]; internal_snprintf(command, sizeof(command), "0x%zx\n", addr); const char *buf = process_->SendCommand(command); diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc index e4ff525d6b05..3fcd7d04880e 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc @@ -24,7 +24,9 @@ #include "sanitizer_symbolizer_libbacktrace.h" #include "sanitizer_symbolizer_mac.h" +#include <dlfcn.h> // for dlsym() #include <errno.h> +#include <stdint.h> #include <stdlib.h> #include <sys/wait.h> #include <unistd.h> @@ -59,6 +61,44 @@ const char *DemangleCXXABI(const char *name) { return name; } +// As of now, there are no headers for the Swift runtime. Once they are +// present, we will weakly link since we do not require Swift runtime to be +// linked. +typedef char *(*swift_demangle_ft)(const char *mangledName, + size_t mangledNameLength, char *outputBuffer, + size_t *outputBufferSize, uint32_t flags); +static swift_demangle_ft swift_demangle_f; + +// This must not happen lazily at symbolication time, because dlsym uses +// malloc and thread-local storage, which is not a good thing to do during +// symbolication. +static void InitializeSwiftDemangler() { + swift_demangle_f = (swift_demangle_ft)dlsym(RTLD_DEFAULT, "swift_demangle"); +} + +// Attempts to demangle a Swift name. The demangler will return nullptr if a +// non-Swift name is passed in. +const char *DemangleSwift(const char *name) { + if (!name) return nullptr; + + // Check if we are dealing with a Swift mangled name first. + if (name[0] != '_' || name[1] != 'T') { + return nullptr; + } + + if (swift_demangle_f) + return swift_demangle_f(name, internal_strlen(name), 0, 0, 0); + + return nullptr; +} + +const char *DemangleSwiftAndCXX(const char *name) { + if (!name) return nullptr; + if (const char *swift_demangled_name = DemangleSwift(name)) + return swift_demangled_name; + return DemangleCXXABI(name); +} + bool SymbolizerProcess::StartSymbolizerSubprocess() { if (!FileExists(path_)) { if (!reported_invalid_path_) { @@ -72,8 +112,15 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() { if (use_forkpty_) { #if SANITIZER_MAC fd_t fd = kInvalidFd; + + // forkpty redirects stdout and stderr into a single stream, so we would + // receive error messages as standard replies. To avoid that, let's dup + // stderr and restore it in the child. + int saved_stderr = dup(STDERR_FILENO); + CHECK_GE(saved_stderr, 0); + // Use forkpty to disable buffering in the new terminal. - pid = forkpty(&fd, 0, 0, 0); + pid = internal_forkpty(&fd); if (pid == -1) { // forkpty() failed. Report("WARNING: failed to fork external symbolizer (errno: %d)\n", @@ -81,6 +128,11 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() { return false; } else if (pid == 0) { // Child subprocess. + + // Restore stderr. + CHECK_GE(dup2(saved_stderr, STDERR_FILENO), 0); + close(saved_stderr); + const char *argv[kArgVMax]; GetArgV(path_, argv); execv(path_, const_cast<char **>(&argv[0])); @@ -90,6 +142,8 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() { // Continue execution in parent process. input_fd_ = output_fd_ = fd; + close(saved_stderr); + // Disable echo in the new terminal, disable CR. struct termios termflags; tcgetattr(fd, &termflags); @@ -135,47 +189,23 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() { CHECK(infd); CHECK(outfd); - // Real fork() may call user callbacks registered with pthread_atfork(). - pid = internal_fork(); - if (pid == -1) { - // Fork() failed. + const char *argv[kArgVMax]; + GetArgV(path_, argv); + pid = StartSubprocess(path_, argv, /* stdin */ outfd[0], + /* stdout */ infd[1]); + if (pid < 0) { internal_close(infd[0]); - internal_close(infd[1]); - internal_close(outfd[0]); internal_close(outfd[1]); - Report("WARNING: failed to fork external symbolizer " - " (errno: %d)\n", errno); return false; - } else if (pid == 0) { - // Child subprocess. - internal_close(STDOUT_FILENO); - internal_close(STDIN_FILENO); - internal_dup2(outfd[0], STDIN_FILENO); - internal_dup2(infd[1], STDOUT_FILENO); - internal_close(outfd[0]); - internal_close(outfd[1]); - internal_close(infd[0]); - internal_close(infd[1]); - for (int fd = sysconf(_SC_OPEN_MAX); fd > 2; fd--) - internal_close(fd); - const char *argv[kArgVMax]; - GetArgV(path_, argv); - execv(path_, const_cast<char **>(&argv[0])); - internal__exit(1); } - // Continue execution in parent process. - internal_close(outfd[0]); - internal_close(infd[1]); input_fd_ = infd[0]; output_fd_ = outfd[1]; } // Check that symbolizer subprocess started successfully. - int pid_status; SleepForMillis(kSymbolizerStartupTimeMillis); - int exited_pid = waitpid(pid, &pid_status, WNOHANG); - if (exited_pid != 0) { + if (!IsProcessRunning(pid)) { // Either waitpid failed, or child has already exited. Report("WARNING: external symbolizer didn't start up correctly!\n"); return false; @@ -372,7 +402,7 @@ class InternalSymbolizer : public SymbolizerTool { #endif // SANITIZER_SUPPORTS_WEAK_HOOKS const char *Symbolizer::PlatformDemangle(const char *name) { - return DemangleCXXABI(name); + return DemangleSwiftAndCXX(name); } void Symbolizer::PlatformPrepareForSandboxing() {} @@ -431,7 +461,9 @@ static void ChooseSymbolizerTools(IntrusiveList<SymbolizerTool> *list, VReport(2, "Symbolizer is disabled.\n"); return; } - if (SymbolizerTool *tool = InternalSymbolizer::get(allocator)) { + if (IsReportingOOM()) { + VReport(2, "Cannot use internal symbolizer: out of memory\n"); + } else if (SymbolizerTool *tool = InternalSymbolizer::get(allocator)) { VReport(2, "Using internal symbolizer.\n"); list->push_back(tool); return; @@ -450,10 +482,6 @@ static void ChooseSymbolizerTools(IntrusiveList<SymbolizerTool> *list, VReport(2, "Using dladdr symbolizer.\n"); list->push_back(new(*allocator) DlAddrSymbolizer()); #endif // SANITIZER_MAC - - if (list->size() == 0) { - Report("WARNING: no internal or external symbolizer found.\n"); - } } Symbolizer *Symbolizer::PlatformInit() { @@ -463,6 +491,11 @@ Symbolizer *Symbolizer::PlatformInit() { return new(symbolizer_allocator_) Symbolizer(list); } +void Symbolizer::LateInitialize() { + Symbolizer::GetOrInit(); + InitializeSwiftDemangler(); +} + } // namespace __sanitizer #endif // SANITIZER_POSIX diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_win.cc b/libsanitizer/sanitizer_common/sanitizer_symbolizer_win.cc index dadb0eaaf6e8..95fda7ec42e0 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_win.cc +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_win.cc @@ -40,6 +40,8 @@ bool TrySymInitialize() { // FIXME: We don't call SymCleanup() on exit yet - should we? } +} // namespace + // Initializes DbgHelp library, if it's not yet initialized. Calls to this // function should be synchronized with respect to other calls to DbgHelp API // (e.g. from WinSymbolizerTool). @@ -95,8 +97,6 @@ void InitializeDbgHelpIfNeeded() { } } -} // namespace - bool WinSymbolizerTool::SymbolizePC(uptr addr, SymbolizedStack *frame) { InitializeDbgHelpIfNeeded(); @@ -277,6 +277,10 @@ Symbolizer *Symbolizer::PlatformInit() { return new(symbolizer_allocator_) Symbolizer(list); } +void Symbolizer::LateInitialize() { + Symbolizer::GetOrInit(); +} + } // namespace __sanitizer #endif // _WIN32 diff --git a/libsanitizer/sanitizer_common/sanitizer_termination.cc b/libsanitizer/sanitizer_common/sanitizer_termination.cc new file mode 100644 index 000000000000..6d7335112b37 --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_termination.cc @@ -0,0 +1,84 @@ +//===-- sanitizer_termination.cc --------------------------------*- C++ -*-===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// This file contains the Sanitizer termination functions CheckFailed and Die, +/// and the callback functionalities associated with them. +/// +//===----------------------------------------------------------------------===// + +#include "sanitizer_common.h" +#include "sanitizer_libc.h" + +namespace __sanitizer { + +static const int kMaxNumOfInternalDieCallbacks = 5; +static DieCallbackType InternalDieCallbacks[kMaxNumOfInternalDieCallbacks]; + +bool AddDieCallback(DieCallbackType callback) { + for (int i = 0; i < kMaxNumOfInternalDieCallbacks; i++) { + if (InternalDieCallbacks[i] == nullptr) { + InternalDieCallbacks[i] = callback; + return true; + } + } + return false; +} + +bool RemoveDieCallback(DieCallbackType callback) { + for (int i = 0; i < kMaxNumOfInternalDieCallbacks; i++) { + if (InternalDieCallbacks[i] == callback) { + internal_memmove(&InternalDieCallbacks[i], &InternalDieCallbacks[i + 1], + sizeof(InternalDieCallbacks[0]) * + (kMaxNumOfInternalDieCallbacks - i - 1)); + InternalDieCallbacks[kMaxNumOfInternalDieCallbacks - 1] = nullptr; + return true; + } + } + return false; +} + +static DieCallbackType UserDieCallback; +void SetUserDieCallback(DieCallbackType callback) { + UserDieCallback = callback; +} + +void NORETURN Die() { + if (UserDieCallback) + UserDieCallback(); + for (int i = kMaxNumOfInternalDieCallbacks - 1; i >= 0; i--) { + if (InternalDieCallbacks[i]) + InternalDieCallbacks[i](); + } + if (common_flags()->abort_on_error) + Abort(); + internal__exit(common_flags()->exitcode); +} + +static CheckFailedCallbackType CheckFailedCallback; +void SetCheckFailedCallback(CheckFailedCallbackType callback) { + CheckFailedCallback = callback; +} + +const int kSecondsToSleepWhenRecursiveCheckFailed = 2; + +void NORETURN CheckFailed(const char *file, int line, const char *cond, + u64 v1, u64 v2) { + static atomic_uint32_t num_calls; + if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) > 10) { + SleepForSeconds(kSecondsToSleepWhenRecursiveCheckFailed); + Trap(); + } + + if (CheckFailedCallback) { + CheckFailedCallback(file, line, cond, v1, v2); + } + Report("Sanitizer CHECK failed: %s:%d %s (%lld, %lld)\n", file, line, cond, + v1, v2); + Die(); +} + +} // namespace __sanitizer diff --git a/libsanitizer/sanitizer_common/sanitizer_thread_registry.cc b/libsanitizer/sanitizer_common/sanitizer_thread_registry.cc index bfa610443c1a..c865d2cad84e 100644 --- a/libsanitizer/sanitizer_common/sanitizer_thread_registry.cc +++ b/libsanitizer/sanitizer_common/sanitizer_thread_registry.cc @@ -129,7 +129,7 @@ u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid, tctx = context_factory_(tid); threads_[tid] = tctx; } else { -#ifndef SANITIZER_GO +#if !SANITIZER_GO Report("%s: Thread limit (%u threads) exceeded. Dying.\n", SanitizerToolName, max_threads_); #else @@ -275,6 +275,8 @@ void ThreadRegistry::StartThread(u32 tid, uptr os_id, void *arg) { } void ThreadRegistry::QuarantinePush(ThreadContextBase *tctx) { + if (tctx->tid == 0) + return; // Don't reuse the main thread. It's a special snowflake. dead_threads_.push_back(tctx); if (dead_threads_.size() <= thread_quarantine_size_) return; diff --git a/libsanitizer/sanitizer_common/sanitizer_tls_get_addr.cc b/libsanitizer/sanitizer_common/sanitizer_tls_get_addr.cc index 7c6ef4f90280..229225d95dd6 100644 --- a/libsanitizer/sanitizer_common/sanitizer_tls_get_addr.cc +++ b/libsanitizer/sanitizer_common/sanitizer_tls_get_addr.cc @@ -76,7 +76,7 @@ void DTLS_Destroy() { DTLS_Deallocate(dtls.dtv, s); } -#if defined(__powerpc64__) +#if defined(__powerpc64__) || defined(__mips__) // This is glibc's TLS_DTV_OFFSET: // "Dynamic thread vector pointers point 0x8000 past the start of each // TLS block." diff --git a/libsanitizer/sanitizer_common/sanitizer_unwind_linux_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_unwind_linux_libcdep.cc index 408c21c913bb..eb1c133e4f4e 100644 --- a/libsanitizer/sanitizer_common/sanitizer_unwind_linux_libcdep.cc +++ b/libsanitizer/sanitizer_common/sanitizer_unwind_linux_libcdep.cc @@ -46,6 +46,11 @@ unwind_backtrace_signal_arch_func unwind_backtrace_signal_arch; #if SANITIZER_ANDROID void SanitizerInitializeUnwinder() { + if (AndroidGetApiLevel() >= ANDROID_LOLLIPOP_MR1) return; + + // Pre-lollipop Android can not unwind through signal handler frames with + // libgcc unwinder, but it has a libcorkscrew.so library with the necessary + // workarounds. void *p = dlopen("libcorkscrew.so", RTLD_LAZY); if (!p) { VReport(1, @@ -101,6 +106,11 @@ _Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context *ctx, void *param) { UnwindTraceArg *arg = (UnwindTraceArg*)param; CHECK_LT(arg->stack->size, arg->max_depth); uptr pc = Unwind_GetIP(ctx); + const uptr kPageSize = GetPageSizeCached(); + // Let's assume that any pointer in the 0th page (i.e. <0x1000 on i386 and + // x86_64) is invalid and stop unwinding here. If we're adding support for + // a platform where this isn't true, we need to reconsider this check. + if (pc < kPageSize) return UNWIND_STOP; arg->stack->trace_buffer[arg->stack->size++] = pc; if (arg->stack->size == arg->max_depth) return UNWIND_STOP; return UNWIND_CONTINUE; diff --git a/libsanitizer/sanitizer_common/sanitizer_win.cc b/libsanitizer/sanitizer_common/sanitizer_win.cc index 0c0a29cea445..785883ce7675 100644 --- a/libsanitizer/sanitizer_common/sanitizer_win.cc +++ b/libsanitizer/sanitizer_common/sanitizer_win.cc @@ -25,7 +25,9 @@ #include "sanitizer_libc.h" #include "sanitizer_mutex.h" #include "sanitizer_placement_new.h" +#include "sanitizer_procmaps.h" #include "sanitizer_stacktrace.h" +#include "sanitizer_symbolizer.h" namespace __sanitizer { @@ -33,13 +35,15 @@ namespace __sanitizer { // --------------------- sanitizer_common.h uptr GetPageSize() { - // FIXME: there is an API for getting the system page size (GetSystemInfo or - // GetNativeSystemInfo), but if we use it here we get test failures elsewhere. - return 1U << 14; + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwPageSize; } uptr GetMmapGranularity() { - return 1U << 16; // FIXME: is this configurable? + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwAllocationGranularity; } uptr GetMaxVirtualAddress() { @@ -81,10 +85,11 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, } #endif // #if !SANITIZER_GO -void *MmapOrDie(uptr size, const char *mem_type) { +void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) { void *rv = VirtualAlloc(0, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); if (rv == 0) - ReportMmapFailureAndDie(size, mem_type, "allocate", GetLastError()); + ReportMmapFailureAndDie(size, mem_type, "allocate", + GetLastError(), raw_report); return rv; } @@ -92,20 +97,90 @@ void UnmapOrDie(void *addr, uptr size) { if (!size || !addr) return; - if (VirtualFree(addr, size, MEM_DECOMMIT) == 0) { - Report("ERROR: %s failed to " - "deallocate 0x%zx (%zd) bytes at address %p (error code: %d)\n", - SanitizerToolName, size, size, addr, GetLastError()); - CHECK("unable to unmap" && 0); + MEMORY_BASIC_INFORMATION mbi; + CHECK(VirtualQuery(addr, &mbi, sizeof(mbi))); + + // MEM_RELEASE can only be used to unmap whole regions previously mapped with + // VirtualAlloc. So we first try MEM_RELEASE since it is better, and if that + // fails try MEM_DECOMMIT. + if (VirtualFree(addr, 0, MEM_RELEASE) == 0) { + if (VirtualFree(addr, size, MEM_DECOMMIT) == 0) { + Report("ERROR: %s failed to " + "deallocate 0x%zx (%zd) bytes at address %p (error code: %d)\n", + SanitizerToolName, size, size, addr, GetLastError()); + CHECK("unable to unmap" && 0); + } } } +// We want to map a chunk of address space aligned to 'alignment'. +void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type) { + CHECK(IsPowerOfTwo(size)); + CHECK(IsPowerOfTwo(alignment)); + + // Windows will align our allocations to at least 64K. + alignment = Max(alignment, GetMmapGranularity()); + + uptr mapped_addr = + (uptr)VirtualAlloc(0, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + if (!mapped_addr) + ReportMmapFailureAndDie(size, mem_type, "allocate aligned", GetLastError()); + + // If we got it right on the first try, return. Otherwise, unmap it and go to + // the slow path. + if (IsAligned(mapped_addr, alignment)) + return (void*)mapped_addr; + if (VirtualFree((void *)mapped_addr, 0, MEM_RELEASE) == 0) + ReportMmapFailureAndDie(size, mem_type, "deallocate", GetLastError()); + + // If we didn't get an aligned address, overallocate, find an aligned address, + // unmap, and try to allocate at that aligned address. + int retries = 0; + const int kMaxRetries = 10; + for (; retries < kMaxRetries && + (mapped_addr == 0 || !IsAligned(mapped_addr, alignment)); + retries++) { + // Overallocate size + alignment bytes. + mapped_addr = + (uptr)VirtualAlloc(0, size + alignment, MEM_RESERVE, PAGE_NOACCESS); + if (!mapped_addr) + ReportMmapFailureAndDie(size, mem_type, "allocate aligned", + GetLastError()); + + // Find the aligned address. + uptr aligned_addr = RoundUpTo(mapped_addr, alignment); + + // Free the overallocation. + if (VirtualFree((void *)mapped_addr, 0, MEM_RELEASE) == 0) + ReportMmapFailureAndDie(size, mem_type, "deallocate", GetLastError()); + + // Attempt to allocate exactly the number of bytes we need at the aligned + // address. This may fail for a number of reasons, in which case we continue + // the loop. + mapped_addr = (uptr)VirtualAlloc((void *)aligned_addr, size, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + } + + // Fail if we can't make this work quickly. + if (retries == kMaxRetries && mapped_addr == 0) + ReportMmapFailureAndDie(size, mem_type, "allocate aligned", GetLastError()); + + return (void *)mapped_addr; +} + void *MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) { // FIXME: is this really "NoReserve"? On Win32 this does not matter much, // but on Win64 it does. - (void)name; // unsupported - void *p = VirtualAlloc((LPVOID)fixed_addr, size, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + (void)name; // unsupported +#if !SANITIZER_GO && SANITIZER_WINDOWS64 + // On asan/Windows64, use MEM_COMMIT would result in error + // 1455:ERROR_COMMITMENT_LIMIT. + // Asan uses exception handler to commit page on demand. + void *p = VirtualAlloc((LPVOID)fixed_addr, size, MEM_RESERVE, PAGE_READWRITE); +#else + void *p = VirtualAlloc((LPVOID)fixed_addr, size, MEM_RESERVE | MEM_COMMIT, + PAGE_READWRITE); +#endif if (p == 0) Report("ERROR: %s failed to " "allocate %p (%zd) bytes at %p (error code: %d)\n", @@ -113,8 +188,18 @@ void *MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) { return p; } +// Memory space mapped by 'MmapFixedOrDie' must have been reserved by +// 'MmapFixedNoAccess'. void *MmapFixedOrDie(uptr fixed_addr, uptr size) { - return MmapFixedNoReserve(fixed_addr, size); + void *p = VirtualAlloc((LPVOID)fixed_addr, size, + MEM_COMMIT, PAGE_READWRITE); + if (p == 0) { + char mem_type[30]; + internal_snprintf(mem_type, sizeof(mem_type), "memory at address 0x%zx", + fixed_addr); + ReportMmapFailureAndDie(size, mem_type, "allocate", GetLastError()); + } + return p; } void *MmapNoReserveOrDie(uptr size, const char *mem_type) { @@ -122,10 +207,10 @@ void *MmapNoReserveOrDie(uptr size, const char *mem_type) { return MmapOrDie(size, mem_type); } -void *MmapNoAccess(uptr fixed_addr, uptr size, const char *name) { +void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) { (void)name; // unsupported void *res = VirtualAlloc((LPVOID)fixed_addr, size, - MEM_RESERVE | MEM_COMMIT, PAGE_NOACCESS); + MEM_RESERVE, PAGE_NOACCESS); if (res == 0) Report("WARNING: %s failed to " "mprotect %p (%zd) bytes at %p (error code: %d)\n", @@ -133,19 +218,28 @@ void *MmapNoAccess(uptr fixed_addr, uptr size, const char *name) { return res; } +void *MmapNoAccess(uptr size) { + void *res = VirtualAlloc(nullptr, size, MEM_RESERVE, PAGE_NOACCESS); + if (res == 0) + Report("WARNING: %s failed to " + "mprotect %p (%zd) bytes (error code: %d)\n", + SanitizerToolName, size, size, GetLastError()); + return res; +} + bool MprotectNoAccess(uptr addr, uptr size) { DWORD old_protection; return VirtualProtect((LPVOID)addr, size, PAGE_NOACCESS, &old_protection); } -void FlushUnneededShadowMemory(uptr addr, uptr size) { +void ReleaseMemoryToOS(uptr addr, uptr size) { // This is almost useless on 32-bits. // FIXME: add madvise-analog when we move to 64-bits. } void NoHugePagesInRegion(uptr addr, uptr size) { - // FIXME: probably similar to FlushUnneededShadowMemory. + // FIXME: probably similar to ReleaseMemoryToOS. } void DontDumpShadowMemory(uptr addr, uptr length) { @@ -153,6 +247,26 @@ void DontDumpShadowMemory(uptr addr, uptr length) { // FIXME: add madvise-analog when we move to 64-bits. } +uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding) { + uptr address = 0; + while (true) { + MEMORY_BASIC_INFORMATION info; + if (!::VirtualQuery((void*)address, &info, sizeof(info))) + return 0; + + if (info.State == MEM_FREE) { + uptr shadow_address = RoundUpTo((uptr)info.BaseAddress + left_padding, + alignment); + if (shadow_address + size < (uptr)info.BaseAddress + info.RegionSize) + return shadow_address; + } + + // Move to the next region. + address = (uptr)info.BaseAddress + info.RegionSize; + } + return 0; +} + bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) { MEMORY_BASIC_INFORMATION mbi; CHECK(VirtualQuery((void *)range_start, &mbi, sizeof(mbi))); @@ -218,7 +332,7 @@ struct ModuleInfo { uptr end_address; }; -#ifndef SANITIZER_GO +#if !SANITIZER_GO int CompareModulesBase(const void *pl, const void *pr) { const ModuleInfo *l = (ModuleInfo *)pl, *r = (ModuleInfo *)pr; if (l->base_address < r->base_address) @@ -228,18 +342,18 @@ int CompareModulesBase(const void *pl, const void *pr) { #endif } // namespace -#ifndef SANITIZER_GO +#if !SANITIZER_GO void DumpProcessMap() { Report("Dumping process modules:\n"); - InternalScopedBuffer<LoadedModule> modules(kMaxNumberOfModules); - uptr num_modules = - GetListOfModules(modules.data(), kMaxNumberOfModules, nullptr); + ListOfModules modules; + modules.init(); + uptr num_modules = modules.size(); InternalScopedBuffer<ModuleInfo> module_infos(num_modules); for (size_t i = 0; i < num_modules; ++i) { module_infos[i].filepath = modules[i].full_name(); - module_infos[i].base_address = modules[i].base_address(); - module_infos[i].end_address = modules[i].ranges().next()->end; + module_infos[i].base_address = modules[i].ranges().front()->beg; + module_infos[i].end_address = modules[i].ranges().back()->end; } qsort(module_infos.data(), num_modules, sizeof(ModuleInfo), CompareModulesBase); @@ -314,6 +428,7 @@ void Abort() { internal__exit(3); } +#if !SANITIZER_GO // Read the file to extract the ImageBase field from the PE header. If ASLR is // disabled and this virtual address is available, the loader will typically // load the image at this address. Therefore, we call it the preferred base. Any @@ -366,9 +481,8 @@ static uptr GetPreferredBase(const char *modname) { return (uptr)pe_header->ImageBase; } -#ifndef SANITIZER_GO -uptr GetListOfModules(LoadedModule *modules, uptr max_modules, - string_predicate_t filter) { +void ListOfModules::init() { + clear(); HANDLE cur_process = GetCurrentProcess(); // Query the list of modules. Start by assuming there are no more than 256 @@ -390,10 +504,8 @@ uptr GetListOfModules(LoadedModule *modules, uptr max_modules, } // |num_modules| is the number of modules actually present, - // |count| is the number of modules we return. - size_t nun_modules = bytes_required / sizeof(HMODULE), - count = 0; - for (size_t i = 0; i < nun_modules && count < max_modules; ++i) { + size_t num_modules = bytes_required / sizeof(HMODULE); + for (size_t i = 0; i < num_modules; ++i) { HMODULE handle = hmodules[i]; MODULEINFO mi; if (!GetModuleInformation(cur_process, handle, &mi, sizeof(mi))) @@ -411,9 +523,6 @@ uptr GetListOfModules(LoadedModule *modules, uptr max_modules, &module_name[0], kMaxPathLength, NULL, NULL); module_name[module_name_len] = '\0'; - if (filter && !filter(module_name)) - continue; - uptr base_address = (uptr)mi.lpBaseOfDll; uptr end_address = (uptr)mi.lpBaseOfDll + mi.SizeOfImage; @@ -424,15 +533,13 @@ uptr GetListOfModules(LoadedModule *modules, uptr max_modules, uptr preferred_base = GetPreferredBase(&module_name[0]); uptr adjusted_base = base_address - preferred_base; - LoadedModule *cur_module = &modules[count]; - cur_module->set(module_name, adjusted_base); + LoadedModule cur_module; + cur_module.set(module_name, adjusted_base); // We add the whole module as one single address range. - cur_module->addAddressRange(base_address, end_address, /*executable*/ true); - count++; + cur_module.addAddressRange(base_address, end_address, /*executable*/ true); + modules_.push_back(cur_module); } UnmapOrDie(hmodules, modules_buffer_size); - - return count; }; // We can't use atexit() directly at __asan_init time as the CRT is not fully @@ -459,14 +566,15 @@ __declspec(allocate(".CRT$XID")) int (*__run_atexit)() = RunAtexit; // ------------------ sanitizer_libc.h fd_t OpenFile(const char *filename, FileAccessMode mode, error_t *last_error) { + // FIXME: Use the wide variants to handle Unicode filenames. fd_t res; if (mode == RdOnly) { - res = CreateFile(filename, GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + res = CreateFileA(filename, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); } else if (mode == WrOnly) { - res = CreateFile(filename, GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, - FILE_ATTRIBUTE_NORMAL, nullptr); + res = CreateFileA(filename, GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, nullptr); } else { UNIMPLEMENTED(); } @@ -613,7 +721,7 @@ void InitTlsSize() { void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, uptr *tls_addr, uptr *tls_size) { -#ifdef SANITIZER_GO +#if SANITIZER_GO *stk_addr = 0; *stk_size = 0; *tls_addr = 0; @@ -634,7 +742,7 @@ void BufferedStackTrace::SlowUnwindStack(uptr pc, u32 max_depth) { // FIXME: CaptureStackBackTrace might be too slow for us. // FIXME: Compare with StackWalk64. // FIXME: Look at LLVMUnhandledExceptionFilter in Signals.inc - size = CaptureStackBackTrace(2, Min(max_depth, kStackTraceMax), + size = CaptureStackBackTrace(1, Min(max_depth, kStackTraceMax), (void**)trace, 0); if (size == 0) return; @@ -649,6 +757,9 @@ void BufferedStackTrace::SlowUnwindStackWithContext(uptr pc, void *context, CONTEXT ctx = *(CONTEXT *)context; STACKFRAME64 stack_frame; memset(&stack_frame, 0, sizeof(stack_frame)); + + InitializeDbgHelpIfNeeded(); + size = 0; #if defined(_WIN64) int machine_type = IMAGE_FILE_MACHINE_AMD64; @@ -697,7 +808,7 @@ void InstallDeadlySignalHandlers(SignalHandlerType handler) { // FIXME: Decide what to do on Windows. } -bool IsDeadlySignal(int signum) { +bool IsHandledDeadlySignal(int signum) { // FIXME: Decide what to do on Windows. return false; } @@ -728,8 +839,8 @@ bool IsAccessibleMemoryRange(uptr beg, uptr size) { } SignalContext SignalContext::Create(void *siginfo, void *context) { - EXCEPTION_RECORD *exception_record = (EXCEPTION_RECORD*)siginfo; - CONTEXT *context_record = (CONTEXT*)context; + EXCEPTION_RECORD *exception_record = (EXCEPTION_RECORD *)siginfo; + CONTEXT *context_record = (CONTEXT *)context; uptr pc = (uptr)exception_record->ExceptionAddress; #ifdef _WIN64 @@ -741,7 +852,19 @@ SignalContext SignalContext::Create(void *siginfo, void *context) { #endif uptr access_addr = exception_record->ExceptionInformation[1]; - return SignalContext(context, access_addr, pc, sp, bp); + // The contents of this array are documented at + // https://msdn.microsoft.com/en-us/library/windows/desktop/aa363082(v=vs.85).aspx + // The first element indicates read as 0, write as 1, or execute as 8. The + // second element is the faulting address. + WriteFlag write_flag = SignalContext::UNKNOWN; + switch (exception_record->ExceptionInformation[0]) { + case 0: write_flag = SignalContext::READ; break; + case 1: write_flag = SignalContext::WRITE; break; + case 8: write_flag = SignalContext::UNKNOWN; break; + } + bool is_memory_access = write_flag != SignalContext::UNKNOWN; + return SignalContext(context, access_addr, pc, sp, bp, is_memory_access, + write_flag); } uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) { @@ -759,6 +882,49 @@ void CheckVMASize() { // Do nothing. } +void MaybeReexec() { + // No need to re-exec on Windows. +} + +char **GetArgv() { + // FIXME: Actually implement this function. + return 0; +} + +pid_t StartSubprocess(const char *program, const char *const argv[], + fd_t stdin_fd, fd_t stdout_fd, fd_t stderr_fd) { + // FIXME: implement on this platform + // Should be implemented based on + // SymbolizerProcess::StarAtSymbolizerSubprocess + // from lib/sanitizer_common/sanitizer_symbolizer_win.cc. + return -1; +} + +bool IsProcessRunning(pid_t pid) { + // FIXME: implement on this platform. + return false; +} + +int WaitForProcess(pid_t pid) { return -1; } + +// FIXME implement on this platform. +void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) { } + + } // namespace __sanitizer +#if !SANITIZER_GO +// Workaround to implement weak hooks on Windows. COFF doesn't directly support +// weak symbols, but it does support /alternatename, which is similar. If the +// user does not override the hook, we will use this default definition instead +// of null. +extern "C" void __sanitizer_print_memory_profile(int top_percent) {} + +#ifdef _WIN64 +#pragma comment(linker, "/alternatename:__sanitizer_print_memory_profile=__sanitizer_default_print_memory_profile") // NOLINT +#else +#pragma comment(linker, "/alternatename:___sanitizer_print_memory_profile=___sanitizer_default_print_memory_profile") // NOLINT +#endif +#endif + #endif // _WIN32 |