From 70aaff4fae73a7e4317ccc4fa99fd6fc4790c61a Mon Sep 17 00:00:00 2001 From: Betul Buyukkurt Date: Wed, 18 Nov 2015 18:12:35 +0000 Subject: [PGO] Runtime support for value profiling. This change adds extends the data structures and adds in the routines for handling runtime calls for value profiling. The profile data format is modified and the version number is incremented. git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@253483 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/profile/InstrProfiling.c | 148 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 147 insertions(+), 1 deletion(-) (limited to 'lib/profile/InstrProfiling.c') diff --git a/lib/profile/InstrProfiling.c b/lib/profile/InstrProfiling.c index 8d010df28..42779acc7 100644 --- a/lib/profile/InstrProfiling.c +++ b/lib/profile/InstrProfiling.c @@ -8,6 +8,8 @@ \*===----------------------------------------------------------------------===*/ #include "InstrProfiling.h" +#include +#include #include __attribute__((visibility("hidden"))) @@ -33,10 +35,15 @@ uint64_t __llvm_profile_get_magic(void) { (uint64_t)129; } +__attribute__((visibility("hidden"))) +uint8_t __llvm_profile_get_num_padding_bytes(uint64_t SizeInBytes) { + return 7 & (sizeof(uint64_t) - SizeInBytes % sizeof(uint64_t)); +} + __attribute__((visibility("hidden"))) uint64_t __llvm_profile_get_version(void) { /* This should be bumped any time the output format changes. */ - return 1; + return 2; } __attribute__((visibility("hidden"))) @@ -45,4 +52,143 @@ void __llvm_profile_reset_counters(void) { uint64_t *E = __llvm_profile_end_counters(); memset(I, 0, sizeof(uint64_t)*(E - I)); + + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + for (const __llvm_profile_data *DI = DataBegin; DI != DataEnd; ++DI) { + if (!DI->ValueCounters) + continue; + + uint64_t CurrentVSiteCount = 0; + for (uint32_t VKI = VK_FIRST; VKI <= VK_LAST; ++VKI) + CurrentVSiteCount += DI->NumValueSites[VKI]; + + for (uint32_t i = 0; i < CurrentVSiteCount; ++i) { + __llvm_profile_value_node *CurrentVNode = DI->ValueCounters[i]; + + while (CurrentVNode) { + CurrentVNode->VData.NumTaken = 0; + CurrentVNode = CurrentVNode->Next; + } + } + } +} + +static uint64_t TotalValueDataSize = 0; + +__attribute__((visibility("hidden"))) +void __llvm_profile_instrument_target(uint64_t TargetValue, void *Data_, + uint32_t CounterIndex) { + + __llvm_profile_data *Data = (__llvm_profile_data*)Data_; + if (!Data) + return; + + if (!Data->ValueCounters) { + uint64_t NumVSites = 0; + for (uint32_t VKI = VK_FIRST; VKI <= VK_LAST; ++VKI) + NumVSites += Data->NumValueSites[VKI]; + + __llvm_profile_value_node** Mem = (__llvm_profile_value_node**) + calloc(NumVSites, sizeof(__llvm_profile_value_node*)); + if (!Mem) + return; + if (!__sync_bool_compare_and_swap(&Data->ValueCounters, 0, Mem)) { + free(Mem); + return; + } + // Acccount for padding during write out. + uint8_t Padding = __llvm_profile_get_num_padding_bytes(NumVSites); + __sync_fetch_and_add(&TotalValueDataSize, NumVSites + Padding); + } + + __llvm_profile_value_node *PrevVNode = NULL; + __llvm_profile_value_node *CurrentVNode = Data->ValueCounters[CounterIndex]; + + uint8_t VDataCount = 0; + while (CurrentVNode) { + if (TargetValue == CurrentVNode->VData.TargetValue) { + CurrentVNode->VData.NumTaken++; + return; + } + PrevVNode = CurrentVNode; + CurrentVNode = CurrentVNode->Next; + ++VDataCount; + } + + if (VDataCount >= UCHAR_MAX) + return; + + CurrentVNode = (__llvm_profile_value_node*) + calloc(1, sizeof(__llvm_profile_value_node)); + if (!CurrentVNode) + return; + + CurrentVNode->VData.TargetValue = TargetValue; + CurrentVNode->VData.NumTaken++; + + uint32_t Success = 0; + if (!Data->ValueCounters[CounterIndex]) + Success = __sync_bool_compare_and_swap( + &(Data->ValueCounters[CounterIndex]), 0, CurrentVNode); + else if (PrevVNode && !PrevVNode->Next) + Success = __sync_bool_compare_and_swap(&(PrevVNode->Next), 0, CurrentVNode); + + if (!Success) { + free(CurrentVNode); + return; + } + __sync_fetch_and_add(&TotalValueDataSize, + Success * sizeof(__llvm_profile_value_data)); +} + +__attribute__((visibility("hidden"))) +uint64_t __llvm_profile_gather_value_data(uint8_t **VDataArray) { + + if (!VDataArray || 0 == TotalValueDataSize) + return 0; + + uint64_t NumData = TotalValueDataSize; + *VDataArray = (uint8_t*) calloc(NumData, sizeof(uint8_t)); + if (!*VDataArray) + return 0; + + uint8_t *VDataEnd = *VDataArray + NumData; + uint8_t *PerSiteCountsHead = *VDataArray; + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + for (__llvm_profile_data *I = (__llvm_profile_data *)DataBegin; + I != DataEnd; ++I) { + + if (!I->ValueCounters) + continue; + + uint64_t NumVSites = 0; + for (uint32_t VKI = VK_FIRST; VKI <= VK_LAST; ++VKI) + NumVSites += I->NumValueSites[VKI]; + uint8_t Padding = __llvm_profile_get_num_padding_bytes(NumVSites); + + uint8_t *PerSiteCountPtr = PerSiteCountsHead; + __llvm_profile_value_data *VDataPtr = + (__llvm_profile_value_data *)(PerSiteCountPtr + NumVSites + Padding); + + for (uint32_t i = 0; i < NumVSites; ++i) { + + __llvm_profile_value_node *VNode = I->ValueCounters[i]; + + uint8_t VDataCount = 0; + while (VNode && ((uint8_t*)(VDataPtr + 1) <= VDataEnd)) { + *VDataPtr = VNode->VData; + VNode = VNode->Next; + ++VDataPtr; + if (++VDataCount == UCHAR_MAX) + break; + } + *PerSiteCountPtr = VDataCount; + ++PerSiteCountPtr; + } + I->ValueCounters = (void *)PerSiteCountsHead; + PerSiteCountsHead = (uint8_t *)VDataPtr; + } + return PerSiteCountsHead - *VDataArray; } -- cgit v1.2.3