summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--lib/sanitizer_common/sanitizer_allocator_primary32.h58
-rw-r--r--lib/sanitizer_common/sanitizer_common.h10
-rw-r--r--lib/sanitizer_common/sanitizer_posix.cc23
-rw-r--r--lib/sanitizer_common/sanitizer_win.cc6
-rw-r--r--lib/sanitizer_common/tests/sanitizer_common_test.cc35
5 files changed, 112 insertions, 20 deletions
diff --git a/lib/sanitizer_common/sanitizer_allocator_primary32.h b/lib/sanitizer_common/sanitizer_allocator_primary32.h
index e13510ba3..bfedbc0a7 100644
--- a/lib/sanitizer_common/sanitizer_allocator_primary32.h
+++ b/lib/sanitizer_common/sanitizer_allocator_primary32.h
@@ -24,7 +24,7 @@ template<class SizeClassAllocator> struct SizeClassAllocator32LocalCache;
// be returned by MmapOrDie().
//
// Region:
-// a result of a single call to MmapAlignedOrDie(kRegionSize, kRegionSize).
+// a result of an allocation of kRegionSize bytes aligned on kRegionSize.
// Since the regions are aligned by kRegionSize, there are exactly
// kNumPossibleRegions possible regions in the address space and so we keep
// a ByteMap possible_regions to store the size classes of each Region.
@@ -106,6 +106,7 @@ class SizeClassAllocator32 {
void Init(s32 release_to_os_interval_ms) {
possible_regions.TestOnlyInit();
internal_memset(size_class_info_array, 0, sizeof(size_class_info_array));
+ num_stashed_regions = 0;
}
s32 ReleaseToOSIntervalMs() const {
@@ -275,15 +276,49 @@ class SizeClassAllocator32 {
return mem & ~(kRegionSize - 1);
}
+ // Allocates a region of kRegionSize bytes, aligned on kRegionSize. If we get
+ // more than one region back (in the event the allocation is aligned on the
+ // first try), attempt to store the second region into a stash. If the stash
+ // is full, just unmap the superfluous memory.
+ uptr AllocateRegionSlow(AllocatorStats *stat) {
+ uptr map_size = kRegionSize;
+ uptr padding_chunk;
+ uptr region = reinterpret_cast<uptr>(
+ MmapAlignedOrDie(kRegionSize, kRegionSize, "SizeClassAllocator32",
+ &padding_chunk));
+ if (padding_chunk) {
+ // We have an extra region, attempt to stash it.
+ CHECK_EQ(padding_chunk, region + kRegionSize);
+ bool trim_extra = true;
+ {
+ SpinMutexLock l(&regions_stash_mutex);
+ if (num_stashed_regions < kMaxStashedRegions) {
+ regions_stash[num_stashed_regions++] = padding_chunk;
+ map_size = 2 * kRegionSize;
+ trim_extra = false;
+ }
+ }
+ if (trim_extra)
+ UnmapOrDie((void*)padding_chunk, kRegionSize);
+ }
+ MapUnmapCallback().OnMap(region, map_size);
+ stat->Add(AllocatorStatMapped, map_size);
+ return region;
+ }
+
uptr AllocateRegion(AllocatorStats *stat, uptr class_id) {
CHECK_LT(class_id, kNumClasses);
- uptr res = reinterpret_cast<uptr>(MmapAlignedOrDie(kRegionSize, kRegionSize,
- "SizeClassAllocator32"));
- MapUnmapCallback().OnMap(res, kRegionSize);
- stat->Add(AllocatorStatMapped, kRegionSize);
- CHECK_EQ(0U, (res & (kRegionSize - 1)));
- possible_regions.set(ComputeRegionId(res), static_cast<u8>(class_id));
- return res;
+ uptr region = 0;
+ {
+ SpinMutexLock l(&regions_stash_mutex);
+ if (num_stashed_regions > 0)
+ region = regions_stash[--num_stashed_regions];
+ }
+ if (!region)
+ region = AllocateRegionSlow(stat);
+ CHECK(IsAligned(region, kRegionSize));
+ possible_regions.set(ComputeRegionId(region), static_cast<u8>(class_id));
+ return region;
}
SizeClassInfo *GetSizeClassInfo(uptr class_id) {
@@ -316,6 +351,13 @@ class SizeClassAllocator32 {
}
}
+ // Unless several threads request regions simultaneously from different size
+ // classes, the stash rarely contains more than 1 entry.
+ static const uptr kMaxStashedRegions = 8;
+ SpinMutex regions_stash_mutex;
+ uptr num_stashed_regions;
+ uptr regions_stash[kMaxStashedRegions];
+
ByteMap possible_regions;
SizeClassInfo size_class_info_array[kNumClasses];
};
diff --git a/lib/sanitizer_common/sanitizer_common.h b/lib/sanitizer_common/sanitizer_common.h
index 79ee6873b..ea9a7d342 100644
--- a/lib/sanitizer_common/sanitizer_common.h
+++ b/lib/sanitizer_common/sanitizer_common.h
@@ -92,7 +92,15 @@ void *MmapFixedOrDie(uptr fixed_addr, uptr size);
void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name = nullptr);
void *MmapNoAccess(uptr size);
// Map aligned chunk of address space; size and alignment are powers of two.
-void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type);
+// Since the predominant use case of this function is "size == alignment" and
+// the nature of the way the alignment requirement is satisfied (by allocating
+// size+alignment bytes of memory), there's a potential of address space
+// fragmentation. The padding_chunk parameter provides the opportunity to
+// return the contiguous padding of "size" bytes of the allocated chunk if the
+// initial allocation happened to be perfectly aligned and the platform supports
+// partial unmapping of the mapped region.
+void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type,
+ uptr *padding_chunk);
// Disallow access to a memory range. Use MmapFixedNoAccess to allocate an
// unaccessible memory.
bool MprotectNoAccess(uptr addr, uptr size);
diff --git a/lib/sanitizer_common/sanitizer_posix.cc b/lib/sanitizer_common/sanitizer_posix.cc
index 9916f4d38..3a6b711db 100644
--- a/lib/sanitizer_common/sanitizer_posix.cc
+++ b/lib/sanitizer_common/sanitizer_posix.cc
@@ -146,22 +146,29 @@ void UnmapOrDie(void *addr, uptr size) {
}
// We want to map a chunk of address space aligned to 'alignment'.
-// We do it by maping a bit more and then unmaping redundant pieces.
+// We do it by mapping a bit more and then unmapping redundant pieces.
// We probably can do it with fewer syscalls in some OS-dependent way.
-void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type) {
+void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type,
+ uptr* padding_chunk) {
CHECK(IsPowerOfTwo(size));
CHECK(IsPowerOfTwo(alignment));
uptr map_size = size + alignment;
uptr map_res = (uptr)MmapOrDie(map_size, mem_type);
uptr map_end = map_res + map_size;
+ bool is_aligned = IsAligned(map_res, alignment);
+ if (is_aligned && padding_chunk && size == alignment) {
+ *padding_chunk = map_res + size;
+ return (void *)map_res;
+ }
+ if (padding_chunk)
+ *padding_chunk = 0;
uptr res = map_res;
- if (res & (alignment - 1)) // Not aligned.
- res = (map_res + alignment) & ~(alignment - 1);
- uptr end = res + size;
- if (res != map_res)
+ if (!is_aligned) {
+ res = (map_res + alignment - 1) & ~(alignment - 1);
UnmapOrDie((void*)map_res, res - map_res);
- if (end != map_end)
- UnmapOrDie((void*)end, map_end - end);
+ }
+ uptr end = res + size;
+ UnmapOrDie((void*)end, map_end - end);
return (void*)res;
}
diff --git a/lib/sanitizer_common/sanitizer_win.cc b/lib/sanitizer_common/sanitizer_win.cc
index afc3bb0ac..493167a19 100644
--- a/lib/sanitizer_common/sanitizer_win.cc
+++ b/lib/sanitizer_common/sanitizer_win.cc
@@ -132,10 +132,14 @@ void UnmapOrDie(void *addr, uptr size) {
}
// We want to map a chunk of address space aligned to 'alignment'.
-void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type) {
+void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type,
+ uptr *padding_chunk) {
CHECK(IsPowerOfTwo(size));
CHECK(IsPowerOfTwo(alignment));
+ if (padding_chunk)
+ *padding_chunk = 0;
+
// Windows will align our allocations to at least 64K.
alignment = Max(alignment, GetMmapGranularity());
diff --git a/lib/sanitizer_common/tests/sanitizer_common_test.cc b/lib/sanitizer_common/tests/sanitizer_common_test.cc
index ebc885db7..c1077e628 100644
--- a/lib/sanitizer_common/tests/sanitizer_common_test.cc
+++ b/lib/sanitizer_common/tests/sanitizer_common_test.cc
@@ -77,8 +77,8 @@ TEST(SanitizerCommon, MmapAlignedOrDie) {
for (uptr size = 1; size <= 32; size *= 2) {
for (uptr alignment = 1; alignment <= 32; alignment *= 2) {
for (int iter = 0; iter < 100; iter++) {
- uptr res = (uptr)MmapAlignedOrDie(
- size * PageSize, alignment * PageSize, "MmapAlignedOrDieTest");
+ uptr res = (uptr)MmapAlignedOrDie(size * PageSize, alignment * PageSize,
+ "MmapAlignedOrDieTest", nullptr);
EXPECT_EQ(0U, res % (alignment * PageSize));
internal_memset((void*)res, 1, size * PageSize);
UnmapOrDie((void*)res, size * PageSize);
@@ -87,6 +87,37 @@ TEST(SanitizerCommon, MmapAlignedOrDie) {
}
}
+TEST(SanitizerCommon, MmapAlignedOrDiePaddingChunk) {
+ uptr PageSize = GetPageSizeCached();
+ for (uptr size = 1; size <= 32; size *= 2) {
+ for (uptr alignment = 1; alignment <= 32; alignment *= 2) {
+ for (int iter = 0; iter < 100; iter++) {
+ uptr padding_chunk;
+ uptr res = (uptr)MmapAlignedOrDie(size * PageSize, alignment * PageSize,
+ "MmapAlignedOrDiePaddingChunkTest", &padding_chunk);
+ EXPECT_EQ(0U, res % (alignment * PageSize));
+ internal_memset((void*)res, 1, size * PageSize);
+ UnmapOrDie((void*)res, size * PageSize);
+ if (SANITIZER_WINDOWS || (size != alignment)) {
+ // Not supported on Windows or for different size and alignment.
+ EXPECT_EQ(0U, padding_chunk);
+ continue;
+ }
+ if (size == 1 && alignment == 1) {
+ // mmap returns PageSize aligned chunks, so this is a specific case
+ // where we can check that padding_chunk will never be 0.
+ EXPECT_NE(0U, padding_chunk);
+ }
+ if (padding_chunk) {
+ EXPECT_EQ(res + size * PageSize, padding_chunk);
+ internal_memset((void*)padding_chunk, 1, alignment * PageSize);
+ UnmapOrDie((void*)padding_chunk, alignment * PageSize);
+ }
+ }
+ }
+ }
+}
+
#if SANITIZER_LINUX
TEST(SanitizerCommon, SanitizerSetThreadName) {
const char *names[] = {
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-04 09:46:36 +0000