summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--lib/profile/InstrProfiling.h9
-rw-r--r--lib/profile/InstrProfilingFile.c16
-rw-r--r--lib/profile/InstrProfilingInternal.h7
-rw-r--r--lib/profile/InstrProfilingValue.c124
-rw-r--r--lib/profile/InstrProfilingWriter.c96
-rw-r--r--test/profile/instrprof-value-prof.c16
6 files changed, 150 insertions, 118 deletions
diff --git a/lib/profile/InstrProfiling.h b/lib/profile/InstrProfiling.h
index c924a42f6..d27ca569d 100644
--- a/lib/profile/InstrProfiling.h
+++ b/lib/profile/InstrProfiling.h
@@ -78,15 +78,16 @@ void INSTR_PROF_VALUE_PROF_FUNC(
/*!
* \brief Prepares the value profiling data for output.
*
- * Prepares a single __llvm_profile_value_data array out of the many
- * ValueProfNode trees (one per instrumented function).
+ * Returns an array of pointers to value profile data.
*/
-uint64_t __llvm_profile_gather_value_data(uint8_t **DataArray);
+struct ValueProfData;
+struct ValueProfData **__llvm_profile_gather_value_data(uint64_t *Size);
/*!
* \brief Write instrumentation data to the current file.
*
- * Writes to the file with the last name given to \a __llvm_profile_set_filename(),
+ * Writes to the file with the last name given to \a *
+ * __llvm_profile_set_filename(),
* or if it hasn't been called, the \c LLVM_PROFILE_FILE environment variable,
* or if that's not set, the last name given to
* \a __llvm_profile_override_default_filename(), or if that's not set,
diff --git a/lib/profile/InstrProfilingFile.c b/lib/profile/InstrProfilingFile.c
index 7f2923c30..a803ba527 100644
--- a/lib/profile/InstrProfilingFile.c
+++ b/lib/profile/InstrProfilingFile.c
@@ -31,12 +31,16 @@ static uint32_t fileWriter(ProfDataIOVec *IOVecs, uint32_t NumIOVecs,
}
static int writeFile(FILE *File) {
- uint8_t *ValueDataBegin = NULL;
- const uint64_t ValueDataSize =
- __llvm_profile_gather_value_data(&ValueDataBegin);
- int r = llvmWriteProfData(fileWriter, File, ValueDataBegin, ValueDataSize);
- free(ValueDataBegin);
- return r;
+ const char *BufferSzStr = 0;
+ uint64_t ValueDataSize = 0;
+ struct ValueProfData **ValueDataArray =
+ __llvm_profile_gather_value_data(&ValueDataSize);
+ FreeHook = &free;
+ CallocHook = &calloc;
+ BufferSzStr = getenv("LLVM_VP_BUFFER_SIZE");
+ if (BufferSzStr && BufferSzStr[0])
+ VPBufferSize = atoi(BufferSzStr);
+ return llvmWriteProfData(fileWriter, File, ValueDataArray, ValueDataSize);
}
static int writeFileWithName(const char *OutputName) {
diff --git a/lib/profile/InstrProfilingInternal.h b/lib/profile/InstrProfilingInternal.h
index e1302d3a6..8edd82c3b 100644
--- a/lib/profile/InstrProfilingInternal.h
+++ b/lib/profile/InstrProfilingInternal.h
@@ -52,17 +52,20 @@ typedef uint32_t (*WriterCallback)(ProfDataIOVec *, uint32_t NumIOVecs,
uint32_t llvmBufferWriter(ProfDataIOVec *IOVecs, uint32_t NumIOVecs,
void **WriterCtx);
int llvmWriteProfData(WriterCallback Writer, void *WriterCtx,
- const uint8_t *ValueDataBegin,
+ struct ValueProfData **ValueDataArray,
const uint64_t ValueDataSize);
int llvmWriteProfDataImpl(WriterCallback Writer, void *WriterCtx,
const __llvm_profile_data *DataBegin,
const __llvm_profile_data *DataEnd,
const uint64_t *CountersBegin,
const uint64_t *CountersEnd,
- const uint8_t *ValueDataBegin,
+ struct ValueProfData **ValueDataBeginArray,
const uint64_t ValueDataSize, const char *NamesBegin,
const char *NamesEnd);
extern char *(*GetEnvHook)(const char *);
+extern void (*FreeHook)(void *);
+extern void* (*CallocHook)(size_t, size_t);
+extern uint32_t VPBufferSize;
#endif
diff --git a/lib/profile/InstrProfilingValue.c b/lib/profile/InstrProfilingValue.c
index 4888eec86..39b4da446 100644
--- a/lib/profile/InstrProfilingValue.c
+++ b/lib/profile/InstrProfilingValue.c
@@ -21,7 +21,8 @@
#define PROF_OOM_RETURN(Msg) \
{ \
PROF_OOM(Msg) \
- return 0; \
+ free(ValueDataArray); \
+ return NULL; \
}
#if COMPILER_RT_HAS_ATOMICS != 1
@@ -78,24 +79,6 @@ static int allocateValueProfileCounters(__llvm_profile_data *Data) {
return 1;
}
-static void deallocateValueProfileCounters(__llvm_profile_data *Data) {
- uint64_t NumVSites = 0, I;
- uint32_t VKI;
- if (!Data->Values)
- return;
- for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
- NumVSites += Data->NumValueSites[VKI];
- for (I = 0; I < NumVSites; I++) {
- ValueProfNode *Node = ((ValueProfNode **)Data->Values)[I];
- while (Node) {
- ValueProfNode *Next = Node->Next;
- free(Node);
- Node = Next;
- }
- }
- free(Data->Values);
-}
-
COMPILER_RT_VISIBILITY void
__llvm_profile_instrument_target(uint64_t TargetValue, void *Data,
uint32_t CounterIndex) {
@@ -147,92 +130,51 @@ __llvm_profile_instrument_target(uint64_t TargetValue, void *Data,
}
}
-/* For multi-threaded programs, while the profile is being dumped, other
- threads may still be updating the value profile data and creating new
- value entries. To accommadate this, we need to add extra bytes to the
- data buffer. The size of the extra space is controlled by an environment
- variable. */
-static unsigned getVprofExtraBytes() {
- const char *ExtraStr =
- GetEnvHook ? GetEnvHook("LLVM_VALUE_PROF_BUFFER_EXTRA") : 0;
- if (!ExtraStr || !ExtraStr[0])
- return 1024;
- return (unsigned)atoi(ExtraStr);
-}
-
-/* Extract the value profile data info from the runtime. */
-#define DEF_VALUE_RECORD(R, NS, V) \
- ValueProfRuntimeRecord R; \
- if (initializeValueProfRuntimeRecord(&R, NS, V)) \
- PROF_OOM_RETURN("Failed to write value profile data ");
-
-#define DTOR_VALUE_RECORD(R) finalizeValueProfRuntimeRecord(&R);
-
-COMPILER_RT_VISIBILITY uint64_t
-__llvm_profile_gather_value_data(uint8_t **VDataArray) {
- size_t S = 0, RealSize = 0, BufferCapacity = 0, Extra = 0;
+COMPILER_RT_VISIBILITY ValueProfData **
+__llvm_profile_gather_value_data(uint64_t *ValueDataSize) {
+ size_t S = 0;
__llvm_profile_data *I;
- if (!VDataArray)
- PROF_OOM_RETURN("Failed to write value profile data ");
+ ValueProfData **ValueDataArray;
const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
+ if (!ValueDataSize)
+ return NULL;
+
+ ValueDataArray =
+ (ValueProfData **)calloc(DataEnd - DataBegin, sizeof(void *));
+ if (!ValueDataArray)
+ PROF_OOM_RETURN("Failed to write value profile data ");
+
/*
* Compute the total Size of the buffer to hold ValueProfData
* structures for functions with value profile data.
*/
for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) {
-
- DEF_VALUE_RECORD(R, I->NumValueSites, I->Values);
+ ValueProfRuntimeRecord R;
+ if (initializeValueProfRuntimeRecord(&R, I->NumValueSites, I->Values))
+ PROF_OOM_RETURN("Failed to write value profile data ");
/* Compute the size of ValueProfData from this runtime record. */
- if (getNumValueKindsRT(&R) != 0)
- S += getValueProfDataSizeRT(&R);
-
- DTOR_VALUE_RECORD(R);
- }
- /* No value sites or no value profile data is collected. */
- if (!S)
- return 0;
-
- Extra = getVprofExtraBytes();
- BufferCapacity = S + Extra;
- *VDataArray = calloc(BufferCapacity, sizeof(uint8_t));
- if (!*VDataArray)
- PROF_OOM_RETURN("Failed to write value profile data ");
-
- ValueProfData *VD = (ValueProfData *)(*VDataArray);
- /*
- * Extract value profile data and write into ValueProfData structure
- * one by one. Note that new value profile data added to any value
- * site (from another thread) after the ValueProfRuntimeRecord is
- * initialized (when the profile data snapshot is taken) won't be
- * collected. This is not a problem as those dropped value will have
- * very low taken count.
- */
- for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) {
- DEF_VALUE_RECORD(R, I->NumValueSites, I->Values);
- if (getNumValueKindsRT(&R) == 0)
- continue;
-
- /* Record R has taken a snapshot of the VP data at this point. Newly
- added VP data for this function will be dropped. */
- /* Check if there is enough space. */
- if (BufferCapacity - RealSize < getValueProfDataSizeRT(&R)) {
- PROF_ERR("Value profile data is dropped :%s \n",
- "Out of buffer space. Use environment "
- " LLVM_VALUE_PROF_BUFFER_EXTRA to allocate more");
- I->Values = 0;
+ if (getNumValueKindsRT(&R) != 0) {
+ ValueProfData *VD = NULL;
+ uint32_t VS = getValueProfDataSizeRT(&R);
+ VD = (ValueProfData *)calloc(VS, sizeof(uint8_t));
+ if (!VD)
+ PROF_OOM_RETURN("Failed to write value profile data ");
+ serializeValueProfDataFromRT(&R, VD);
+ ValueDataArray[I - DataBegin] = VD;
+ S += VS;
}
+ finalizeValueProfRuntimeRecord(&R);
+ }
- serializeValueProfDataFromRT(&R, VD);
- deallocateValueProfileCounters(I);
- I->Values = VD;
- RealSize += VD->TotalSize;
- VD = (ValueProfData *)((char *)VD + VD->TotalSize);
- DTOR_VALUE_RECORD(R);
+ if (!S) {
+ free(ValueDataArray);
+ ValueDataArray = NULL;
}
- return RealSize;
+ *ValueDataSize = S;
+ return ValueDataArray;
}
diff --git a/lib/profile/InstrProfilingWriter.c b/lib/profile/InstrProfilingWriter.c
index 8519891f2..6153a0050 100644
--- a/lib/profile/InstrProfilingWriter.c
+++ b/lib/profile/InstrProfilingWriter.c
@@ -11,6 +11,12 @@
#include "InstrProfilingInternal.h"
#include <string.h>
+#define INSTR_PROF_VALUE_PROF_DATA
+#include "InstrProfData.inc"
+void (*FreeHook)(void *) = NULL;
+void* (*CallocHook)(size_t, size_t) = NULL;
+uint32_t VPBufferSize = 0;
+
/* The buffer writer is reponsponsible in keeping writer state
* across the call.
*/
@@ -29,7 +35,7 @@ COMPILER_RT_VISIBILITY uint32_t llvmBufferWriter(ProfDataIOVec *IOVecs,
COMPILER_RT_VISIBILITY int llvmWriteProfData(WriterCallback Writer,
void *WriterCtx,
- const uint8_t *ValueDataBegin,
+ ValueProfData **ValueDataArray,
const uint64_t ValueDataSize) {
/* Match logic in __llvm_profile_write_buffer(). */
const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
@@ -39,15 +45,80 @@ COMPILER_RT_VISIBILITY int llvmWriteProfData(WriterCallback Writer,
const char *NamesBegin = __llvm_profile_begin_names();
const char *NamesEnd = __llvm_profile_end_names();
return llvmWriteProfDataImpl(Writer, WriterCtx, DataBegin, DataEnd,
- CountersBegin, CountersEnd, ValueDataBegin,
+ CountersBegin, CountersEnd, ValueDataArray,
ValueDataSize, NamesBegin, NamesEnd);
}
+#define VP_BUFFER_SIZE 8 * 1024
+static int writeValueProfData(WriterCallback Writer, void *WriterCtx,
+ ValueProfData **ValueDataBegin,
+ uint64_t NumVData) {
+ ValueProfData **ValueDataArray = ValueDataBegin;
+ char *BufferStart = 0, *Buffer;
+ ValueProfData *CurVData;
+ uint32_t I = 0, BufferSz;
+
+ if (!ValueDataBegin)
+ return 0;
+
+ BufferSz = VPBufferSize ? VPBufferSize : VP_BUFFER_SIZE;
+ BufferStart = (char *)CallocHook(BufferSz, sizeof(uint8_t));
+ if (!BufferStart)
+ return -1;
+
+ uint32_t WriteSize = 0;
+ Buffer = BufferStart;
+ do {
+ CurVData = ValueDataArray[I];
+ if (!CurVData) {
+ I++;
+ continue;
+ }
+
+ /* Buffer is full or not large enough, it is time to flush. */
+ if (CurVData->TotalSize + WriteSize > BufferSz) {
+ if (WriteSize) {
+ ProfDataIOVec IO[] = {{BufferStart, sizeof(uint8_t), WriteSize}};
+ if (Writer(IO, 1, &WriterCtx))
+ return -1;
+ WriteSize = 0;
+ Buffer = BufferStart;
+ }
+ /* Special case, bypass the buffer completely. */
+ if (CurVData->TotalSize > BufferSz) {
+ ProfDataIOVec IO[] = {{CurVData, sizeof(uint8_t), CurVData->TotalSize}};
+ if (Writer(IO, 1, &WriterCtx))
+ return -1;
+ FreeHook(ValueDataArray[I]);
+ I++;
+ }
+ } else {
+ /* Write the data to buffer */
+ ProfDataIOVec IO[] = {{CurVData, sizeof(uint8_t), CurVData->TotalSize}};
+ llvmBufferWriter(IO, 1, (void **)&Buffer);
+ WriteSize += CurVData->TotalSize;
+ FreeHook(ValueDataArray[I]);
+ I++;
+ }
+ } while (I < NumVData);
+
+ /* Final flush. */
+ if (WriteSize) {
+ ProfDataIOVec IO[] = {{BufferStart, sizeof(uint8_t), WriteSize}};
+ if (Writer(IO, 1, &WriterCtx))
+ return -1;
+ }
+
+ FreeHook(ValueDataBegin);
+ FreeHook(BufferStart);
+ return 0;
+}
+
COMPILER_RT_VISIBILITY int llvmWriteProfDataImpl(
WriterCallback Writer, void *WriterCtx,
const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
const uint64_t *CountersBegin, const uint64_t *CountersEnd,
- const uint8_t *ValueDataBegin, const uint64_t ValueDataSize,
+ ValueProfData **ValueDataBegin, const uint64_t ValueDataSize,
const char *NamesBegin, const char *NamesEnd) {
/* Calculate size of sections. */
@@ -70,18 +141,13 @@ COMPILER_RT_VISIBILITY int llvmWriteProfDataImpl(
#include "InstrProfData.inc"
/* Write the data. */
- ProfDataIOVec IOVec[] = {
- {&Header, sizeof(__llvm_profile_header), 1},
- {DataBegin, sizeof(__llvm_profile_data), DataSize},
- {CountersBegin, sizeof(uint64_t), CountersSize},
- {NamesBegin, sizeof(char), NamesSize},
- {Zeroes, sizeof(char), Padding}};
+ ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1},
+ {DataBegin, sizeof(__llvm_profile_data), DataSize},
+ {CountersBegin, sizeof(uint64_t), CountersSize},
+ {NamesBegin, sizeof(uint8_t), NamesSize},
+ {Zeroes, sizeof(uint8_t), Padding}};
if (Writer(IOVec, sizeof(IOVec) / sizeof(*IOVec), &WriterCtx))
return -1;
- if (ValueDataBegin) {
- ProfDataIOVec IOVec2[] = {{ValueDataBegin, sizeof(char), ValueDataSize}};
- if (Writer(IOVec2, sizeof(IOVec2) / sizeof(*IOVec2), &WriterCtx))
- return -1;
- }
- return 0;
+
+ return writeValueProfData(Writer, WriterCtx, ValueDataBegin, DataSize);
}
diff --git a/test/profile/instrprof-value-prof.c b/test/profile/instrprof-value-prof.c
index db2a1a7b3..3ccecbe75 100644
--- a/test/profile/instrprof-value-prof.c
+++ b/test/profile/instrprof-value-prof.c
@@ -9,6 +9,22 @@
// value profile merging current do sorting based on target values -- this will destroy the order of the target
// in the list leading to comparison problem. For now just check a small subset of output.
// RUN: llvm-profdata show --all-functions -ic-targets %t-merged.profdata | FileCheck %s -check-prefix=MERGE
+//
+// RUN: env LLVM_PROFILE_FILE=%t-3.profraw LLVM_VP_BUFFER_SIZE=1 %run %t 1
+// RUN: env LLVM_PROFILE_FILE=%t-4.profraw LLVM_VP_BUFFER_SIZE=8 %run %t 1
+// RUN: env LLVM_PROFILE_FILE=%t-5.profraw LLVM_VP_BUFFER_SIZE=128 %run %t 1
+// RUN: env LLVM_PROFILE_FILE=%t-6.profraw LLVM_VP_BUFFER_SIZE=1024 %run %t 1
+// RUN: env LLVM_PROFILE_FILE=%t-7.profraw LLVM_VP_BUFFER_SIZE=102400 %run %t 1
+// RUN: llvm-profdata merge -o %t-3.profdata %t-3.profraw
+// RUN: llvm-profdata merge -o %t-4.profdata %t-4.profraw
+// RUN: llvm-profdata merge -o %t-5.profdata %t-5.profraw
+// RUN: llvm-profdata merge -o %t-6.profdata %t-6.profraw
+// RUN: llvm-profdata merge -o %t-7.profdata %t-7.profraw
+// RUN: llvm-profdata show --all-functions -ic-targets %t-3.profdata | FileCheck %s
+// RUN: llvm-profdata show --all-functions -ic-targets %t-4.profdata | FileCheck %s
+// RUN: llvm-profdata show --all-functions -ic-targets %t-5.profdata | FileCheck %s
+// RUN: llvm-profdata show --all-functions -ic-targets %t-6.profdata | FileCheck %s
+// RUN: llvm-profdata show --all-functions -ic-targets %t-7.profdata | FileCheck %s
#include <stdint.h>
#include <stdio.h>
generated by cgit v1.2.3 (git 2.25.1) at 2024-05-19 09:14:09 +0000