summaryrefslogtreecommitdiff
path: root/lib/hwasan/hwasan_allocator.cc
diff options
context:
space:
mode:
Diffstat (limited to 'lib/hwasan/hwasan_allocator.cc')
-rw-r--r--lib/hwasan/hwasan_allocator.cc330
1 files changed, 330 insertions, 0 deletions
diff --git a/lib/hwasan/hwasan_allocator.cc b/lib/hwasan/hwasan_allocator.cc
new file mode 100644
index 000000000..fbcaf78b8
--- /dev/null
+++ b/lib/hwasan/hwasan_allocator.cc
@@ -0,0 +1,330 @@
+//===-- hwasan_allocator.cc --------------------------- ---------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// HWAddressSanitizer allocator.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_checks.h"
+#include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_errno.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "hwasan.h"
+#include "hwasan_allocator.h"
+#include "hwasan_thread.h"
+#include "hwasan_poisoning.h"
+
+namespace __hwasan {
+
+enum {
+ CHUNK_INVALID = 0,
+ CHUNK_FREE = 1,
+ CHUNK_ALLOCATED = 2
+};
+
+struct Metadata {
+ u64 state : 2;
+ u64 requested_size : 62;
+ u32 alloc_context_id;
+ u32 free_context_id;
+};
+
+bool HwasanChunkView::IsValid() const {
+ return metadata_ && metadata_->state != CHUNK_INVALID;
+}
+bool HwasanChunkView::IsAllocated() const {
+ return metadata_ && metadata_->state == CHUNK_ALLOCATED;
+}
+uptr HwasanChunkView::Beg() const {
+ return block_;
+}
+uptr HwasanChunkView::End() const {
+ return Beg() + UsedSize();
+}
+uptr HwasanChunkView::UsedSize() const {
+ return metadata_->requested_size;
+}
+u32 HwasanChunkView::GetAllocStackId() const {
+ return metadata_->alloc_context_id;
+}
+u32 HwasanChunkView::GetFreeStackId() const {
+ return metadata_->free_context_id;
+}
+
+struct HwasanMapUnmapCallback {
+ void OnMap(uptr p, uptr size) const {}
+ void OnUnmap(uptr p, uptr size) const {
+ // We are about to unmap a chunk of user memory.
+ // It can return as user-requested mmap() or another thread stack.
+ // Make it accessible with zero-tagged pointer.
+ TagMemory(p, size, 0);
+ }
+};
+
+#if !defined(__aarch64__)
+#error unsupported platform
+#endif
+
+static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G
+static const uptr kRegionSizeLog = 20;
+static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog;
+typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap;
+
+struct AP32 {
+ static const uptr kSpaceBeg = 0;
+ static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
+ static const uptr kMetadataSize = sizeof(Metadata);
+ typedef __sanitizer::CompactSizeClassMap SizeClassMap;
+ static const uptr kRegionSizeLog = __hwasan::kRegionSizeLog;
+ typedef __hwasan::ByteMap ByteMap;
+ typedef HwasanMapUnmapCallback MapUnmapCallback;
+ static const uptr kFlags = 0;
+};
+typedef SizeClassAllocator32<AP32> PrimaryAllocator;
+typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache;
+typedef LargeMmapAllocator<HwasanMapUnmapCallback> SecondaryAllocator;
+typedef CombinedAllocator<PrimaryAllocator, AllocatorCache,
+ SecondaryAllocator> Allocator;
+
+static Allocator allocator;
+static AllocatorCache fallback_allocator_cache;
+static SpinMutex fallback_mutex;
+static atomic_uint8_t hwasan_allocator_tagging_enabled;
+
+void HwasanAllocatorInit() {
+ atomic_store_relaxed(&hwasan_allocator_tagging_enabled,
+ !flags()->disable_allocator_tagging);
+ SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
+ allocator.Init(common_flags()->allocator_release_to_os_interval_ms);
+}
+
+AllocatorCache *GetAllocatorCache(HwasanThreadLocalMallocStorage *ms) {
+ CHECK(ms);
+ CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache));
+ return reinterpret_cast<AllocatorCache *>(ms->allocator_cache);
+}
+
+void HwasanThreadLocalMallocStorage::CommitBack() {
+ allocator.SwallowCache(GetAllocatorCache(this));
+}
+
+static void *HwasanAllocate(StackTrace *stack, uptr size, uptr alignment,
+ bool zeroise) {
+ alignment = Max(alignment, kShadowAlignment);
+ size = RoundUpTo(size, kShadowAlignment);
+
+ if (size > kMaxAllowedMallocSize) {
+ Report("WARNING: HWAddressSanitizer failed to allocate %p bytes\n",
+ (void *)size);
+ return Allocator::FailureHandler::OnBadRequest();
+ }
+ HwasanThread *t = GetCurrentThread();
+ void *allocated;
+ if (t) {
+ AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
+ allocated = allocator.Allocate(cache, size, alignment);
+ } else {
+ SpinMutexLock l(&fallback_mutex);
+ AllocatorCache *cache = &fallback_allocator_cache;
+ allocated = allocator.Allocate(cache, size, alignment);
+ }
+ Metadata *meta =
+ reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated));
+ meta->state = CHUNK_ALLOCATED;
+ meta->requested_size = size;
+ meta->alloc_context_id = StackDepotPut(*stack);
+ if (zeroise)
+ internal_memset(allocated, 0, size);
+
+ void *user_ptr = (flags()->tag_in_malloc &&
+ atomic_load_relaxed(&hwasan_allocator_tagging_enabled))
+ ? (void *)TagMemoryAligned((uptr)allocated, size, 0xBB)
+ : allocated;
+
+ HWASAN_MALLOC_HOOK(user_ptr, size);
+ return user_ptr;
+}
+
+void HwasanDeallocate(StackTrace *stack, void *user_ptr) {
+ CHECK(user_ptr);
+ HWASAN_FREE_HOOK(user_ptr);
+
+ void *p = GetAddressFromPointer(user_ptr);
+ Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p));
+ uptr size = meta->requested_size;
+ meta->state = CHUNK_FREE;
+ meta->requested_size = 0;
+ meta->free_context_id = StackDepotPut(*stack);
+ // This memory will not be reused by anyone else, so we are free to keep it
+ // poisoned.
+ if (flags()->tag_in_free &&
+ atomic_load_relaxed(&hwasan_allocator_tagging_enabled))
+ TagMemoryAligned((uptr)p, size, 0xBC);
+ HwasanThread *t = GetCurrentThread();
+ if (t) {
+ AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
+ allocator.Deallocate(cache, p);
+ } else {
+ SpinMutexLock l(&fallback_mutex);
+ AllocatorCache *cache = &fallback_allocator_cache;
+ allocator.Deallocate(cache, p);
+ }
+}
+
+void *HwasanReallocate(StackTrace *stack, void *user_old_p, uptr new_size,
+ uptr alignment) {
+ alignment = Max(alignment, kShadowAlignment);
+ new_size = RoundUpTo(new_size, kShadowAlignment);
+
+ void *old_p = GetAddressFromPointer(user_old_p);
+ Metadata *meta = reinterpret_cast<Metadata*>(allocator.GetMetaData(old_p));
+ uptr old_size = meta->requested_size;
+ uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(old_p);
+ if (new_size <= actually_allocated_size) {
+ // We are not reallocating here.
+ // FIXME: update stack trace for the allocation?
+ meta->requested_size = new_size;
+ if (!atomic_load_relaxed(&hwasan_allocator_tagging_enabled))
+ return user_old_p;
+ if (flags()->retag_in_realloc)
+ return (void *)TagMemoryAligned((uptr)old_p, new_size, 0xCC);
+ if (new_size > old_size) {
+ tag_t tag = GetTagFromPointer((uptr)user_old_p);
+ TagMemoryAligned((uptr)old_p + old_size, new_size - old_size, tag);
+ }
+ return user_old_p;
+ }
+ uptr memcpy_size = Min(new_size, old_size);
+ void *new_p = HwasanAllocate(stack, new_size, alignment, false /*zeroise*/);
+ if (new_p) {
+ internal_memcpy(new_p, old_p, memcpy_size);
+ HwasanDeallocate(stack, old_p);
+ }
+ return new_p;
+}
+
+HwasanChunkView FindHeapChunkByAddress(uptr address) {
+ void *block = allocator.GetBlockBegin(reinterpret_cast<void*>(address));
+ if (!block)
+ return HwasanChunkView();
+ Metadata *metadata =
+ reinterpret_cast<Metadata*>(allocator.GetMetaData(block));
+ return HwasanChunkView(reinterpret_cast<uptr>(block), metadata);
+}
+
+static uptr AllocationSize(const void *user_ptr) {
+ const void *p = GetAddressFromPointer(user_ptr);
+ if (!p) return 0;
+ const void *beg = allocator.GetBlockBegin(p);
+ if (beg != p) return 0;
+ Metadata *b = (Metadata *)allocator.GetMetaData(p);
+ return b->requested_size;
+}
+
+void *hwasan_malloc(uptr size, StackTrace *stack) {
+ return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false));
+}
+
+void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack) {
+ if (UNLIKELY(CheckForCallocOverflow(size, nmemb)))
+ return SetErrnoOnNull(Allocator::FailureHandler::OnBadRequest());
+ return SetErrnoOnNull(HwasanAllocate(stack, nmemb * size, sizeof(u64), true));
+}
+
+void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack) {
+ if (!ptr)
+ return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false));
+ if (size == 0) {
+ HwasanDeallocate(stack, ptr);
+ return nullptr;
+ }
+ return SetErrnoOnNull(HwasanReallocate(stack, ptr, size, sizeof(u64)));
+}
+
+void *hwasan_valloc(uptr size, StackTrace *stack) {
+ return SetErrnoOnNull(HwasanAllocate(stack, size, GetPageSizeCached(), false));
+}
+
+void *hwasan_pvalloc(uptr size, StackTrace *stack) {
+ uptr PageSize = GetPageSizeCached();
+ if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
+ errno = errno_ENOMEM;
+ return Allocator::FailureHandler::OnBadRequest();
+ }
+ // pvalloc(0) should allocate one page.
+ size = size ? RoundUpTo(size, PageSize) : PageSize;
+ return SetErrnoOnNull(HwasanAllocate(stack, size, PageSize, false));
+}
+
+void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack) {
+ if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
+ errno = errno_EINVAL;
+ return Allocator::FailureHandler::OnBadRequest();
+ }
+ return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, false));
+}
+
+void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack) {
+ if (UNLIKELY(!IsPowerOfTwo(alignment))) {
+ errno = errno_EINVAL;
+ return Allocator::FailureHandler::OnBadRequest();
+ }
+ return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, false));
+}
+
+int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size,
+ StackTrace *stack) {
+ if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
+ Allocator::FailureHandler::OnBadRequest();
+ return errno_EINVAL;
+ }
+ void *ptr = HwasanAllocate(stack, size, alignment, false);
+ if (UNLIKELY(!ptr))
+ return errno_ENOMEM;
+ CHECK(IsAligned((uptr)ptr, alignment));
+ *memptr = ptr;
+ return 0;
+}
+
+} // namespace __hwasan
+
+using namespace __hwasan;
+
+void __hwasan_enable_allocator_tagging() {
+ atomic_store_relaxed(&hwasan_allocator_tagging_enabled, 1);
+}
+
+void __hwasan_disable_allocator_tagging() {
+ atomic_store_relaxed(&hwasan_allocator_tagging_enabled, 0);
+}
+
+uptr __sanitizer_get_current_allocated_bytes() {
+ uptr stats[AllocatorStatCount];
+ allocator.GetStats(stats);
+ return stats[AllocatorStatAllocated];
+}
+
+uptr __sanitizer_get_heap_size() {
+ uptr stats[AllocatorStatCount];
+ allocator.GetStats(stats);
+ return stats[AllocatorStatMapped];
+}
+
+uptr __sanitizer_get_free_bytes() { return 1; }
+
+uptr __sanitizer_get_unmapped_bytes() { return 1; }
+
+uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
+
+int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }
+
+uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }