summaryrefslogtreecommitdiff
path: root/lib/scudo/scudo_allocator.cpp
diff options
context:
space:
mode:
authorKostya Kortchinsky <kostyak@google.com>2017-05-09 15:12:38 +0000
committerKostya Kortchinsky <kostyak@google.com>2017-05-09 15:12:38 +0000
commitb5a980955aabf4d683d37ebbc9d48a724428786b (patch)
treeb82d37f759f534469fc7bbbb249ef3d254cd37c1 /lib/scudo/scudo_allocator.cpp
parente12d24d3e1b367c601b6b9b6f45ede69ff03deee (diff)
[scudo] CRC32 optimizations
Summary: This change optimizes several aspects of the checksum used for chunk headers. First, there is no point in checking the weak symbol `computeHardwareCRC32` everytime, it will either be there or not when we start, so check it once during initialization and set the checksum type accordingly. Then, the loading of `HashAlgorithm` for SSE versions (and ARM equivalent) was not optimized out, while not necessary. So I reshuffled that part of the code, which duplicates a tiny bit of code, but ends up in a much cleaner assembly (and faster as we avoid an extraneous load and some calls). The following code is the checksum at the end of `scudoMalloc` for x86_64 with full SSE 4.2, before: ``` mov rax, 0FFFFFFFFFFFFFFh shl r10, 38h mov edi, dword ptr cs:_ZN7__scudoL6CookieE ; __scudo::Cookie and r14, rax lea rsi, [r13-10h] movzx eax, cs:_ZN7__scudoL13HashAlgorithmE ; __scudo::HashAlgorithm or r14, r10 mov rbx, r14 xor bx, bx call _ZN7__scudo20computeHardwareCRC32Ejm ; __scudo::computeHardwareCRC32(uint,ulong) mov rsi, rbx mov edi, eax call _ZN7__scudo20computeHardwareCRC32Ejm ; __scudo::computeHardwareCRC32(uint,ulong) mov r14w, ax mov rax, r13 mov [r13-10h], r14 ``` After: ``` mov rax, cs:_ZN7__scudoL6CookieE ; __scudo::Cookie lea rcx, [rbx-10h] mov rdx, 0FFFFFFFFFFFFFFh and r14, rdx shl r9, 38h or r14, r9 crc32 eax, rcx mov rdx, r14 xor dx, dx mov eax, eax crc32 eax, rdx mov r14w, ax mov rax, rbx mov [rbx-10h], r14 ``` Reviewers: dvyukov, alekseyshl, kcc Reviewed By: alekseyshl Subscribers: aemerson, rengolin, llvm-commits Differential Revision: https://reviews.llvm.org/D32971 git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@302538 91177308-0d34-0410-b5e6-96231b3b80d8
Diffstat (limited to 'lib/scudo/scudo_allocator.cpp')
-rw-r--r--lib/scudo/scudo_allocator.cpp48
1 files changed, 27 insertions, 21 deletions
diff --git a/lib/scudo/scudo_allocator.cpp b/lib/scudo/scudo_allocator.cpp
index 5420fc964..2b7f099df 100644
--- a/lib/scudo/scudo_allocator.cpp
+++ b/lib/scudo/scudo_allocator.cpp
@@ -15,6 +15,7 @@
//===----------------------------------------------------------------------===//
#include "scudo_allocator.h"
+#include "scudo_crc32.h"
#include "scudo_tls.h"
#include "scudo_utils.h"
@@ -34,21 +35,28 @@ static uptr Cookie;
// at compilation or at runtime.
static atomic_uint8_t HashAlgorithm = { CRC32Software };
-SANITIZER_WEAK_ATTRIBUTE u32 computeHardwareCRC32(u32 Crc, uptr Data);
-
-INLINE u32 computeCRC32(u32 Crc, uptr Data, u8 HashType) {
- // If SSE4.2 is defined here, it was enabled everywhere, as opposed to only
- // for scudo_crc32.cpp. This means that other SSE instructions were likely
- // emitted at other places, and as a result there is no reason to not use
- // the hardware version of the CRC32.
+INLINE u32 computeCRC32(uptr Crc, uptr Value, uptr *Array, uptr ArraySize) {
+ // If the hardware CRC32 feature is defined here, it was enabled everywhere,
+ // as opposed to only for scudo_crc32.cpp. This means that other hardware
+ // specific instructions were likely emitted at other places, and as a
+ // result there is no reason to not use it here.
#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32)
- return computeHardwareCRC32(Crc, Data);
+ Crc = CRC32_INTRINSIC(Crc, Value);
+ for (uptr i = 0; i < ArraySize; i++)
+ Crc = CRC32_INTRINSIC(Crc, Array[i]);
+ return Crc;
#else
- if (computeHardwareCRC32 && HashType == CRC32Hardware)
- return computeHardwareCRC32(Crc, Data);
- else
- return computeSoftwareCRC32(Crc, Data);
-#endif // defined(__SSE4_2__)
+ if (atomic_load_relaxed(&HashAlgorithm) == CRC32Hardware) {
+ Crc = computeHardwareCRC32(Crc, Value);
+ for (uptr i = 0; i < ArraySize; i++)
+ Crc = computeHardwareCRC32(Crc, Array[i]);
+ return Crc;
+ }
+ Crc = computeSoftwareCRC32(Crc, Value);
+ for (uptr i = 0; i < ArraySize; i++)
+ Crc = computeSoftwareCRC32(Crc, Array[i]);
+ return Crc;
+#endif // defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32)
}
static ScudoBackendAllocator &getBackendAllocator();
@@ -78,10 +86,8 @@ struct ScudoChunk : UnpackedHeader {
ZeroChecksumHeader.Checksum = 0;
uptr HeaderHolder[sizeof(UnpackedHeader) / sizeof(uptr)];
memcpy(&HeaderHolder, &ZeroChecksumHeader, sizeof(HeaderHolder));
- u8 HashType = atomic_load_relaxed(&HashAlgorithm);
- u32 Crc = computeCRC32(Cookie, reinterpret_cast<uptr>(this), HashType);
- for (uptr i = 0; i < ARRAY_SIZE(HeaderHolder); i++)
- Crc = computeCRC32(Crc, HeaderHolder[i], HashType);
+ u32 Crc = computeCRC32(Cookie, reinterpret_cast<uptr>(this), HeaderHolder,
+ ARRAY_SIZE(HeaderHolder));
return static_cast<u16>(Crc);
}
@@ -195,10 +201,10 @@ void initScudo() {
CHECK(!ScudoInitIsRunning && "Scudo init calls itself!");
ScudoInitIsRunning = true;
- // Check is SSE4.2 is supported, if so, opt for the CRC32 hardware version.
- if (testCPUFeature(CRC32CPUFeature)) {
+ // Check if hardware CRC32 is supported in the binary and by the platform, if
+ // so, opt for the CRC32 hardware version of the checksum.
+ if (computeHardwareCRC32 && testCPUFeature(CRC32CPUFeature))
atomic_store_relaxed(&HashAlgorithm, CRC32Hardware);
- }
initFlags();
@@ -228,7 +234,7 @@ struct QuarantineCallback {
getBackendAllocator().Deallocate(Cache_, Ptr);
}
- /// Internal quarantine allocation and deallocation functions.
+ // Internal quarantine allocation and deallocation functions.
void *Allocate(uptr Size) {
// TODO(kostyak): figure out the best way to protect the batches.
return getBackendAllocator().Allocate(Cache_, Size, MinAlignment);