[XRay][profiler] Part 4: Profiler Mode Wiring

Summary:
This is part of the larger XRay Profiling Mode effort.

This patch implements the wiring required to enable us to actually
select the `xray-profiling` mode, and install the handlers to start
measuring the time and frequency of the function calls in call stacks.
The current way to get the profile information is by working with the
XRay API to `__xray_process_buffers(...)`.

In subsequent changes we'll implement profile saving to files, similar
to how the FDR and basic modes operate, as well as means for converting
this format into those that can be loaded/visualised as flame graphs. We
will also be extending the accounting tool in LLVM to support
stack-based function call accounting.

We also continue with the implementation to support building small
histograms of latencies for the `FunctionCallTrie::Node` type, to allow
us to actually approximate the distribution of latencies per function.

Depends on D45758 and D46998.

Reviewers: eizan, kpw, pelikan

Reviewed By: kpw

Subscribers: llvm-commits, mgorny

Differential Revision: https://reviews.llvm.org/D44620

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@334469 91177308-0d34-0410-b5e6-96231b3b80d8

10 files changed, 345 insertions, 52 deletions

diff --git a/lib/xray/CMakeLists.txt b/lib/xray/CMakeLists.txt
index 7ac391415..baac739d4 100644
--- a/lib/xray/CMakeLists.txt
+++ b/lib/xray/CMakeLists.txt
@@ -18,9 +18,10 @@ set(XRAY_BASIC_MODE_SOURCES
     xray_basic_flags.cc
     xray_basic_logging.cc)
 
-set(XRAY_PROFILER_MODE_SOURCES
+set(XRAY_PROFILING_MODE_SOURCES
     xray_profile_collector.cc
-    xray_profiler_flags.cc)
+    xray_profiling.cc
+    xray_profiling_flags.cc)
 
 # Implementation files for all XRay architectures.
 set(x86_64_SOURCES
@@ -105,10 +106,10 @@ if (APPLE)
     SOURCES ${XRAY_BASIC_MODE_SOURCES}
     CFLAGS ${XRAY_CFLAGS}
     DEFS ${XRAY_COMMON_DEFINITIONS})
-  add_compiler_rt_object_libraries(RTXrayPROFILER
+  add_compiler_rt_object_libraries(RTXrayPROFILING
     OS ${XRAY_SUPPORTED_OS}
     ARCHS ${XRAY_SUPPORTED_ARCH}
-    SOURCES ${XRAY_PROFILER_MODE_SOURCES}
+    SOURCES ${XRAY_PROFILING_MODE_SOURCES}
     CFLAGS ${XRAY_CFLAGS}
     DEFS ${XRAY_COMMON_DEFINITIONS})
 
@@ -145,11 +146,11 @@ if (APPLE)
     LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS} ${WEAK_SYMBOL_LINK_FLAGS}
     LINK_LIBS ${XRAY_LINK_LIBS}
     PARENT_TARGET xray)
-  add_compiler_rt_runtime(clang_rt.xray-profiler
+  add_compiler_rt_runtime(clang_rt.xray-profiling
     STATIC
     OS ${XRAY_SUPPORTED_OS}
     ARCHS ${XRAY_SUPPORTED_ARCH}
-    OBJECT_LIBS RTXrayPROFILER
+    OBJECT_LIBS RTXrayPROFILING
     CFLAGS ${XRAY_CFLAGS}
     DEFS ${XRAY_COMMON_DEFINITIONS}
     LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS} ${WEAK_SYMBOL_LINK_FLAGS}
@@ -172,9 +173,9 @@ else() # not Apple
       ARCHS ${arch}
       SOURCES ${XRAY_BASIC_MODE_SOURCES} CFLAGS ${XRAY_CFLAGS}
       DEFS ${XRAY_COMMON_DEFINITIONS})
-    add_compiler_rt_object_libraries(RTXrayPROFILER
+    add_compiler_rt_object_libraries(RTXrayPROFILING
       ARCHS ${arch}
-      SOURCES ${XRAY_PROFILER_MODE_SOURCES} CFLAGS ${XRAY_CFLAGS}
+      SOURCES ${XRAY_PROFILING_MODE_SOURCES} CFLAGS ${XRAY_CFLAGS}
       DEFS ${XRAY_COMMON_DEFINITIONS})
 
     # Common XRay archive for instrumented binaries.
@@ -201,13 +202,14 @@ else() # not Apple
       DEFS ${XRAY_COMMON_DEFINITIONS}
       OBJECT_LIBS RTXrayBASIC
       PARENT_TARGET xray)
-    add_compiler_rt_runtime(clang_rt.xray-profiler
-      STATIC
-      ARCHS ${arch}
-      CFLAGS ${XRAY_CFLAGS}
-      DEFS ${XRAY_COMMON_DEFINITIONS}
-      OBJECT_LIBS RTXrayPROFILER
-      PARENT_TARGET xray)
+   # Profiler Mode runtime
+   add_compiler_rt_runtime(clang_rt.xray-profiling
+     STATIC
+     ARCHS ${arch}
+     CFLAGS ${XRAY_CFLAGS}
+     DEFS ${XRAY_COMMON_DEFINITIONS}
+     OBJECT_LIBS RTXrayPROFILING
+     PARENT_TARGET xray)
   endforeach()
 endif() # not Apple
 
diff --git a/lib/xray/tests/CMakeLists.txt b/lib/xray/tests/CMakeLists.txt
index 2a65e118b..eb17b095d 100644
--- a/lib/xray/tests/CMakeLists.txt
+++ b/lib/xray/tests/CMakeLists.txt
@@ -33,8 +33,8 @@ set(XRAY_IMPL_FILES
   ../../xray_powerpc64.cc
   ../../xray_profile_collector.cc
   ../../xray_profile_collector.h
-  ../../xray_profiler_flags.cc
-  ../../xray_profiler_flags.h
+  ../../xray_profiling_flags.cc
+  ../../xray_profiling_flags.h
   ../../xray_recursion_guard.h
   ../../xray_segmented_array.h
   ../../xray_trampoline_powerpc64.cc
@@ -106,7 +106,7 @@ if(COMPILER_RT_CAN_EXECUTE_TESTS)
     add_xray_lib("RTXRay.test.osx"
       $<TARGET_OBJECTS:RTXray.osx>
       $<TARGET_OBJECTS:RTXrayFDR.osx>
-      $<TARGET_OBJECTS:RTXrayPROFILER.osx>
+      $<TARGET_OBJECTS:RTXrayPROFILING.osx>
       $<TARGET_OBJECTS:RTSanitizerCommon.osx>
       $<TARGET_OBJECTS:RTSanitizerCommonLibc.osx>)
   else()
@@ -114,7 +114,7 @@ if(COMPILER_RT_CAN_EXECUTE_TESTS)
     add_xray_lib("RTXRay.test.${arch}"
       $<TARGET_OBJECTS:RTXray.${arch}>
       $<TARGET_OBJECTS:RTXrayFDR.${arch}>
-      $<TARGET_OBJECTS:RTXrayPROFILER.${arch}>
+      $<TARGET_OBJECTS:RTXrayPROFILING.${arch}>
       $<TARGET_OBJECTS:RTSanitizerCommon.${arch}>
       $<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}>)
   endforeach()
diff --git a/lib/xray/tests/unit/function_call_trie_test.cc b/lib/xray/tests/unit/function_call_trie_test.cc
index 3aaa98804..d4a79d902 100644
--- a/lib/xray/tests/unit/function_call_trie_test.cc
+++ b/lib/xray/tests/unit/function_call_trie_test.cc
@@ -27,13 +27,13 @@ TEST(FunctionCallTrieTest, Construction) {
 TEST(FunctionCallTrieTest, ConstructWithTLSAllocators) {
   // FIXME: Support passing in configuration for allocators in the allocator
   // constructors.
-  profilerFlags()->setDefaults();
+  profilingFlags()->setDefaults();
   FunctionCallTrie::Allocators Allocators = FunctionCallTrie::InitAllocators();
   FunctionCallTrie Trie(Allocators);
 }
 
 TEST(FunctionCallTrieTest, EnterAndExitFunction) {
-  profilerFlags()->setDefaults();
+  profilingFlags()->setDefaults();
   auto A = FunctionCallTrie::InitAllocators();
   FunctionCallTrie Trie(A);
 
@@ -71,7 +71,7 @@ TEST(FunctionCallTrieTest, MissingFunctionExit) {
 }
 
 TEST(FunctionCallTrieTest, MultipleRoots) {
-  profilerFlags()->setDefaults();
+  profilingFlags()->setDefaults();
   auto A = FunctionCallTrie::InitAllocators();
   FunctionCallTrie Trie(A);
 
@@ -114,7 +114,7 @@ TEST(FunctionCallTrieTest, MultipleRoots) {
 // accounting local time to `f2` from d = (t3 - t2), then local time to `f1`
 // as d' = (t3 - t1) - d, and then local time to `f0` as d'' = (t3 - t0) - d'.
 TEST(FunctionCallTrieTest, MissingIntermediaryExit) {
-  profilerFlags()->setDefaults();
+  profilingFlags()->setDefaults();
   auto A = FunctionCallTrie::InitAllocators();
   FunctionCallTrie Trie(A);
 
@@ -156,7 +156,7 @@ TEST(FunctionCallTrieTest, MissingIntermediaryExit) {
 // TODO: Test that we can handle cross-CPU migrations, where TSCs are not
 // guaranteed to be synchronised.
 TEST(FunctionCallTrieTest, DeepCopy) {
-  profilerFlags()->setDefaults();
+  profilingFlags()->setDefaults();
   auto A = FunctionCallTrie::InitAllocators();
   FunctionCallTrie Trie(A);
 
@@ -197,7 +197,7 @@ TEST(FunctionCallTrieTest, DeepCopy) {
 }
 
 TEST(FunctionCallTrieTest, MergeInto) {
-  profilerFlags()->setDefaults();
+  profilingFlags()->setDefaults();
   auto A = FunctionCallTrie::InitAllocators();
   FunctionCallTrie T0(A);
   FunctionCallTrie T1(A);
diff --git a/lib/xray/tests/unit/profile_collector_test.cc b/lib/xray/tests/unit/profile_collector_test.cc
index 69003dc48..b7dbe5673 100644
--- a/lib/xray/tests/unit/profile_collector_test.cc
+++ b/lib/xray/tests/unit/profile_collector_test.cc
@@ -13,7 +13,7 @@
 #include "gtest/gtest.h"
 
 #include "xray_profile_collector.h"
-#include "xray_profiler_flags.h"
+#include "xray_profiling_flags.h"
 #include <cstdint>
 #include <thread>
 #include <utility>
@@ -25,7 +25,7 @@ namespace {
 static constexpr auto kHeaderSize = 16u;
 
 void ValidateBlock(XRayBuffer B) {
-  profilerFlags()->setDefaults();
+  profilingFlags()->setDefaults();
   ASSERT_NE(static_cast<const void *>(B.Data), nullptr);
   ASSERT_NE(B.Size, 0u);
   ASSERT_GE(B.Size, kHeaderSize);
@@ -84,7 +84,7 @@ std::tuple<Profile, const char *> ParseProfile(const char *P) {
 }
 
 TEST(profileCollectorServiceTest, PostSerializeCollect) {
-  profilerFlags()->setDefaults();
+  profilingFlags()->setDefaults();
   // The most basic use-case (the one we actually only care about) is the one
   // where we ensure that we can post FunctionCallTrie instances, which are then
   // destroyed but serialized properly.
@@ -157,7 +157,7 @@ void threadProcessing() {
 }
 
 TEST(profileCollectorServiceTest, PostSerializeCollectMultipleThread) {
-  profilerFlags()->setDefaults();
+  profilingFlags()->setDefaults();
   std::thread t1(threadProcessing);
   std::thread t2(threadProcessing);
 
diff --git a/lib/xray/xray_function_call_trie.h b/lib/xray/xray_function_call_trie.h
index 95f760f78..4bff5daff 100644
--- a/lib/xray/xray_function_call_trie.h
+++ b/lib/xray/xray_function_call_trie.h
@@ -15,7 +15,7 @@
 #ifndef XRAY_FUNCTION_CALL_TRIE_H
 #define XRAY_FUNCTION_CALL_TRIE_H
 
-#include "xray_profiler_flags.h"
+#include "xray_profiling_flags.h"
 #include "xray_segmented_array.h"
 #include <utility>
 #include <memory>  // For placement new.
@@ -223,26 +223,26 @@ public:
     auto NodeAllocator = reinterpret_cast<Allocators::NodeAllocatorType *>(
         InternalAlloc(sizeof(Allocators::NodeAllocatorType)));
     new (NodeAllocator) Allocators::NodeAllocatorType(
-        profilerFlags()->per_thread_allocator_max, 0);
+        profilingFlags()->per_thread_allocator_max, 0);
     A.NodeAllocator = NodeAllocator;
 
     auto RootAllocator = reinterpret_cast<Allocators::RootAllocatorType *>(
         InternalAlloc(sizeof(Allocators::RootAllocatorType)));
     new (RootAllocator) Allocators::RootAllocatorType(
-        profilerFlags()->per_thread_allocator_max, 0);
+        profilingFlags()->per_thread_allocator_max, 0);
     A.RootAllocator = RootAllocator;
 
     auto ShadowStackAllocator =
         reinterpret_cast<Allocators::ShadowStackAllocatorType *>(
             InternalAlloc(sizeof(Allocators::ShadowStackAllocatorType)));
     new (ShadowStackAllocator) Allocators::ShadowStackAllocatorType(
-        profilerFlags()->per_thread_allocator_max, 0);
+        profilingFlags()->per_thread_allocator_max, 0);
     A.ShadowStackAllocator = ShadowStackAllocator;
 
     auto NodeIdPairAllocator = reinterpret_cast<NodeIdPairAllocatorType *>(
         InternalAlloc(sizeof(NodeIdPairAllocatorType)));
     new (NodeIdPairAllocator)
-        NodeIdPairAllocatorType(profilerFlags()->per_thread_allocator_max, 0);
+        NodeIdPairAllocatorType(profilingFlags()->per_thread_allocator_max, 0);
     A.NodeIdPairAllocator = NodeIdPairAllocator;
     return A;
   }
@@ -360,7 +360,7 @@ public:
       using Stack = Array<NodeAndParent>;
 
       typename Stack::AllocatorType StackAllocator(
-          profilerFlags()->stack_allocator_max, 0);
+          profilingFlags()->stack_allocator_max, 0);
       Stack DFSStack(StackAllocator);
 
       // TODO: Figure out what to do if we fail to allocate any more stack
@@ -398,7 +398,7 @@ public:
     };
     using Stack = Array<NodeAndTarget>;
     typename Stack::AllocatorType StackAllocator(
-        profilerFlags()->stack_allocator_max, 0);
+        profilingFlags()->stack_allocator_max, 0);
     Stack DFSStack(StackAllocator);
 
     for (const auto Root : getRoots()) {
diff --git a/lib/xray/xray_profile_collector.cc b/lib/xray/xray_profile_collector.cc
index b153affd8..9d057863e 100644
--- a/lib/xray/xray_profile_collector.cc
+++ b/lib/xray/xray_profile_collector.cc
@@ -15,7 +15,7 @@
 #include "xray_profile_collector.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_vector.h"
-#include "xray_profiler_flags.h"
+#include "xray_profiling_flags.h"
 #include <pthread.h>
 #include <memory>
 #include <utility>
@@ -128,7 +128,7 @@ static void populateRecords(ProfileRecordArray &PRs,
                             const FunctionCallTrie &Trie) {
   using StackArray = Array<const FunctionCallTrie::Node *>;
   using StackAllocator = typename StackArray::AllocatorType;
-  StackAllocator StackAlloc(profilerFlags()->stack_allocator_max, 0);
+  StackAllocator StackAlloc(profilingFlags()->stack_allocator_max, 0);
   StackArray DFSStack(StackAlloc);
   for (const auto R : Trie.getRoots()) {
     DFSStack.Append(R);
@@ -198,9 +198,9 @@ void serialize() {
   // Then repopulate the global ProfileBuffers.
   for (u32 I = 0; I < ThreadTries.Size(); ++I) {
     using ProfileRecordAllocator = typename ProfileRecordArray::AllocatorType;
-    ProfileRecordAllocator PRAlloc(profilerFlags()->global_allocator_max, 0);
+    ProfileRecordAllocator PRAlloc(profilingFlags()->global_allocator_max, 0);
     ProfileRecord::PathAllocator PathAlloc(
-        profilerFlags()->global_allocator_max, 0);
+        profilingFlags()->global_allocator_max, 0);
     ProfileRecordArray ProfileRecords(PRAlloc);
 
     // First, we want to compute the amount of space we're going to need. We'll
diff --git a/lib/xray/xray_profiling.cc b/lib/xray/xray_profiling.cc
new file mode 100644
index 000000000..1e7922c1e
--- /dev/null
+++ b/lib/xray/xray_profiling.cc
@@ -0,0 +1,291 @@
+//===-- xray_profiling.cc ---------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of XRay, a dynamic runtime instrumentation system.
+//
+// This is the implementation of a profiling handler.
+//
+//===----------------------------------------------------------------------===//
+#include <memory>
+
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_flags.h"
+#include "xray/xray_interface.h"
+#include "xray/xray_log_interface.h"
+
+#include "xray_flags.h"
+#include "xray_profile_collector.h"
+#include "xray_profiling_flags.h"
+#include "xray_recursion_guard.h"
+#include "xray_tsc.h"
+#include "xray_utils.h"
+#include <pthread.h>
+
+namespace __xray {
+
+namespace {
+
+atomic_sint32_t ProfilerLogFlushStatus = {
+    XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING};
+
+atomic_sint32_t ProfilerLogStatus = {XRayLogInitStatus::XRAY_LOG_UNINITIALIZED};
+
+SpinMutex ProfilerOptionsMutex;
+
+struct alignas(64) ProfilingData {
+  FunctionCallTrie::Allocators *Allocators = nullptr;
+  FunctionCallTrie *FCT = nullptr;
+};
+
+static pthread_key_t ProfilingKey;
+
+ProfilingData &getThreadLocalData() XRAY_NEVER_INSTRUMENT {
+  thread_local std::aligned_storage<sizeof(ProfilingData)>::type ThreadStorage;
+  if (pthread_getspecific(ProfilingKey) == NULL) {
+    new (&ThreadStorage) ProfilingData{};
+    pthread_setspecific(ProfilingKey, &ThreadStorage);
+  }
+
+  auto &TLD = *reinterpret_cast<ProfilingData *>(&ThreadStorage);
+
+  // We need to check whether the global flag to finalizing/finalized has been
+  // switched. If it is, then we ought to not actually initialise the data.
+  auto Status = atomic_load(&ProfilerLogStatus, memory_order_acquire);
+  if (Status == XRayLogInitStatus::XRAY_LOG_FINALIZING ||
+      Status == XRayLogInitStatus::XRAY_LOG_FINALIZED)
+    return TLD;
+
+  // If we're live, then we re-initialize TLD if the pointers are not null.
+  if (UNLIKELY(TLD.Allocators == nullptr && TLD.FCT == nullptr)) {
+    TLD.Allocators = reinterpret_cast<FunctionCallTrie::Allocators *>(
+        InternalAlloc(sizeof(FunctionCallTrie::Allocators)));
+    new (TLD.Allocators) FunctionCallTrie::Allocators();
+    *TLD.Allocators = FunctionCallTrie::InitAllocators();
+    TLD.FCT = reinterpret_cast<FunctionCallTrie *>(
+        InternalAlloc(sizeof(FunctionCallTrie)));
+    new (TLD.FCT) FunctionCallTrie(*TLD.Allocators);
+  }
+
+  return TLD;
+}
+
+} // namespace
+
+const char *profilingCompilerDefinedFlags() XRAY_NEVER_INSTRUMENT {
+#ifdef XRAY_PROFILER_DEFAULT_OPTIONS
+  return SANITIZER_STRINGIFY(XRAY_PROFILER_DEFAULT_OPTIONS);
+#else
+  return "";
+#endif
+}
+
+atomic_sint32_t ProfileFlushStatus = {
+    XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING};
+
+XRayLogFlushStatus profilingFlush() XRAY_NEVER_INSTRUMENT {
+  // When flushing, all we really do is reset the global state, and only when
+  // the log has already been finalized.
+  if (atomic_load(&ProfilerLogStatus, memory_order_acquire) !=
+      XRayLogInitStatus::XRAY_LOG_FINALIZED) {
+    if (Verbosity())
+      Report("Not flushing profiles, profiling not been finalized.\n");
+    return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING;
+  }
+
+  s32 Result = XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING;
+  if (!atomic_compare_exchange_strong(&ProfilerLogFlushStatus, &Result,
+                                      XRayLogFlushStatus::XRAY_LOG_FLUSHING,
+                                      memory_order_acq_rel)) {
+    if (Verbosity())
+      Report("Not flushing profiles, implementation still finalizing.\n");
+  }
+
+  profileCollectorService::reset();
+
+  atomic_store(&ProfilerLogStatus, XRayLogFlushStatus::XRAY_LOG_FLUSHED,
+               memory_order_release);
+
+  return XRayLogFlushStatus::XRAY_LOG_FLUSHED;
+}
+
+namespace {
+
+thread_local atomic_uint8_t ReentranceGuard{0};
+
+void postCurrentThreadFCT(ProfilingData &TLD) {
+  if (TLD.Allocators == nullptr || TLD.FCT == nullptr)
+    return;
+
+  profileCollectorService::post(*TLD.FCT, GetTid());
+  TLD.FCT->~FunctionCallTrie();
+  TLD.Allocators->~Allocators();
+  InternalFree(TLD.FCT);
+  InternalFree(TLD.Allocators);
+  TLD.FCT = nullptr;
+  TLD.Allocators = nullptr;
+}
+
+} // namespace
+
+void profilingHandleArg0(int32_t FuncId,
+                         XRayEntryType Entry) XRAY_NEVER_INSTRUMENT {
+  unsigned char CPU;
+  auto TSC = readTSC(CPU);
+  RecursionGuard G(ReentranceGuard);
+  if (!G)
+    return;
+
+  auto Status = atomic_load(&ProfilerLogStatus, memory_order_acquire);
+  auto &TLD = getThreadLocalData();
+  if (UNLIKELY(Status == XRayLogInitStatus::XRAY_LOG_FINALIZED ||
+               Status == XRayLogInitStatus::XRAY_LOG_FINALIZING)) {
+    postCurrentThreadFCT(TLD);
+    return;
+  }
+
+  switch (Entry) {
+  case XRayEntryType::ENTRY:
+  case XRayEntryType::LOG_ARGS_ENTRY:
+    TLD.FCT->enterFunction(FuncId, TSC);
+    break;
+  case XRayEntryType::EXIT:
+  case XRayEntryType::TAIL:
+    TLD.FCT->exitFunction(FuncId, TSC);
+    break;
+  default:
+    // FIXME: Handle bugs.
+    break;
+  }
+}
+
+void profilingHandleArg1(int32_t FuncId, XRayEntryType Entry,
+                         uint64_t) XRAY_NEVER_INSTRUMENT {
+  return profilingHandleArg0(FuncId, Entry);
+}
+
+XRayLogInitStatus profilingFinalize() XRAY_NEVER_INSTRUMENT {
+  s32 CurrentStatus = XRayLogInitStatus::XRAY_LOG_INITIALIZED;
+  if (!atomic_compare_exchange_strong(&ProfilerLogStatus, &CurrentStatus,
+                                      XRayLogInitStatus::XRAY_LOG_FINALIZING,
+                                      memory_order_release)) {
+    if (Verbosity())
+      Report("Cannot finalize profile, the profiling is not initialized.\n");
+    return static_cast<XRayLogInitStatus>(CurrentStatus);
+  }
+
+  // Wait a grace period to allow threads to see that we're finalizing.
+  SleepForMillis(profilingFlags()->grace_period_ms);
+
+  // We also want to make sure that the current thread's data is cleaned up,
+  // if we have any.
+  auto &TLD = getThreadLocalData();
+  postCurrentThreadFCT(TLD);
+
+  // Then we force serialize the log data.
+  profileCollectorService::serialize();
+
+  atomic_store(&ProfilerLogStatus, XRayLogInitStatus::XRAY_LOG_FINALIZED,
+               memory_order_release);
+  return XRayLogInitStatus::XRAY_LOG_FINALIZED;
+}
+
+XRayLogInitStatus
+profilingLoggingInit(size_t BufferSize, size_t BufferMax, void *Options,
+                     size_t OptionsSize) XRAY_NEVER_INSTRUMENT {
+  if (BufferSize != 0 || BufferMax != 0) {
+    if (Verbosity())
+      Report("__xray_log_init() being used, and is unsupported. Use "
+             "__xray_log_init_mode(...) instead. Bailing out.");
+    return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED;
+  }
+
+  s32 CurrentStatus = XRayLogInitStatus::XRAY_LOG_UNINITIALIZED;
+  if (!atomic_compare_exchange_strong(&ProfilerLogStatus, &CurrentStatus,
+                                      XRayLogInitStatus::XRAY_LOG_INITIALIZING,
+                                      memory_order_release)) {
+    if (Verbosity())
+      Report("Cannot initialize already initialised profiling "
+             "implementation.\n");
+    return static_cast<XRayLogInitStatus>(CurrentStatus);
+  }
+
+  {
+    SpinMutexLock Lock(&ProfilerOptionsMutex);
+    FlagParser ConfigParser;
+    auto *F = profilingFlags();
+    F->setDefaults();
+    registerProfilerFlags(&ConfigParser, F);
+    const char *ProfilerCompileFlags = profilingCompilerDefinedFlags();
+    ConfigParser.ParseString(ProfilerCompileFlags);
+    ConfigParser.ParseString(static_cast<const char *>(Options));
+    if (Verbosity())
+      ReportUnrecognizedFlags();
+  }
+
+  // We need to reset the profile data collection implementation now.
+  profileCollectorService::reset();
+
+  // We need to set up the at-thread-exit handler.
+  static pthread_once_t Once = PTHREAD_ONCE_INIT;
+  pthread_once(&Once, +[] {
+    pthread_key_create(&ProfilingKey, +[](void *P) {
+      // This is the thread-exit handler.
+      auto &TLD = *reinterpret_cast<ProfilingData *>(P);
+      if (TLD.Allocators == nullptr && TLD.FCT == nullptr)
+        return;
+
+      postCurrentThreadFCT(TLD);
+    });
+  });
+
+  __xray_log_set_buffer_iterator(profileCollectorService::nextBuffer);
+  __xray_set_handler(profilingHandleArg0);
+  __xray_set_handler_arg1(profilingHandleArg1);
+
+  atomic_store(&ProfilerLogStatus, XRayLogInitStatus::XRAY_LOG_INITIALIZED,
+               memory_order_release);
+  if (Verbosity())
+    Report("XRay Profiling init successful.\n");
+
+  return XRayLogInitStatus::XRAY_LOG_INITIALIZED;
+}
+
+bool profilingDynamicInitializer() XRAY_NEVER_INSTRUMENT {
+  // Set up the flag defaults from the static defaults and the
+  // compiler-provided defaults.
+  {
+    SpinMutexLock Lock(&ProfilerOptionsMutex);
+    auto *F = profilingFlags();
+    F->setDefaults();
+    FlagParser ProfilingParser;
+    registerProfilerFlags(&ProfilingParser, F);
+    const char *ProfilerCompileFlags = profilingCompilerDefinedFlags();
+    ProfilingParser.ParseString(ProfilerCompileFlags);
+  }
+
+  XRayLogImpl Impl{
+      profilingLoggingInit,
+      profilingFinalize,
+      profilingHandleArg0,
+      profilingFlush,
+  };
+  auto RegistrationResult = __xray_log_register_mode("xray-profiling", Impl);
+  if (RegistrationResult != XRayLogRegisterStatus::XRAY_REGISTRATION_OK &&
+      Verbosity())
+    Report("Cannot register XRay Profiling mode to 'xray-profiling'; error = "
+           "%d\n",
+           RegistrationResult);
+  if (!internal_strcmp(flags()->xray_mode, "xray-profiling"))
+    __xray_set_log_impl(Impl);
+  return true;
+}
+
+} // namespace __xray
+
+static auto UNUSED Unused = __xray::profilingDynamicInitializer();
diff --git a/lib/xray/xray_profiler_flags.cc b/lib/xray/xray_profiling_flags.cc
index 7dc20cedf..593e66a78 100644
--- a/lib/xray/xray_profiler_flags.cc
+++ b/lib/xray/xray_profiling_flags.cc
@@ -12,7 +12,7 @@
 // XRay runtime flags.
 //===----------------------------------------------------------------------===//
 
-#include "xray_profiler_flags.h"
+#include "xray_profiling_flags.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_flag_parser.h"
 #include "sanitizer_common/sanitizer_libc.h"
@@ -20,12 +20,12 @@
 
 namespace __xray {
 
-// Storage for the profiler flags.
-ProfilerFlags xray_profiler_flags_dont_use_directly;
+// Storage for the profiling flags.
+ProfilerFlags xray_profiling_flags_dont_use_directly;
 
 void ProfilerFlags::setDefaults() XRAY_NEVER_INSTRUMENT {
 #define XRAY_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
-#include "xray_profiler_flags.inc"
+#include "xray_profiling_flags.inc"
 #undef XRAY_FLAG
 }
 
@@ -33,7 +33,7 @@ void registerProfilerFlags(FlagParser *P,
                            ProfilerFlags *F) XRAY_NEVER_INSTRUMENT {
 #define XRAY_FLAG(Type, Name, DefaultValue, Description)                       \
   RegisterFlag(P, #Name, Description, &F->Name);
-#include "xray_profiler_flags.inc"
+#include "xray_profiling_flags.inc"
 #undef XRAY_FLAG
 }
 
diff --git a/lib/xray/xray_profiler_flags.h b/lib/xray/xray_profiling_flags.h
index 9a57abb26..2f9a75147 100644
--- a/lib/xray/xray_profiler_flags.h
+++ b/lib/xray/xray_profiling_flags.h
@@ -1,4 +1,4 @@
-//===-- xray_profiler_flags.h ----------------------------------*- C++ -*-===//
+//===-- xray_profiling_flags.h ----------------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -9,7 +9,7 @@
 //
 // This file is a part of XRay, a dynamic runtime instrumentation system.
 //
-// XRay profiler runtime flags.
+// XRay profiling runtime flags.
 //===----------------------------------------------------------------------===//
 
 #ifndef XRAY_PROFILER_FLAGS_H
@@ -22,15 +22,15 @@ namespace __xray {
 
 struct ProfilerFlags {
 #define XRAY_FLAG(Type, Name, DefaultValue, Description) Type Name;
-#include "xray_profiler_flags.inc"
+#include "xray_profiling_flags.inc"
 #undef XRAY_FLAG
 
   void setDefaults();
 };
 
-extern ProfilerFlags xray_profiler_flags_dont_use_directly;
-inline ProfilerFlags *profilerFlags() {
-  return &xray_profiler_flags_dont_use_directly;
+extern ProfilerFlags xray_profiling_flags_dont_use_directly;
+inline ProfilerFlags *profilingFlags() {
+  return &xray_profiling_flags_dont_use_directly;
 }
 void registerProfilerFlags(FlagParser *P, ProfilerFlags *F);
 
diff --git a/lib/xray/xray_profiler_flags.inc b/lib/xray/xray_profiling_flags.inc
index 9266598a1..6381a0748 100644
--- a/lib/xray/xray_profiler_flags.inc
+++ b/lib/xray/xray_profiling_flags.inc
@@ -1,4 +1,4 @@
-//===-- xray_flags.inc ------------------------------------------*- C++ -*-===//
+//===-- xray_profiling_flags.inc --------------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //