Performance improvement.

Using fwrite and fread was very *very* slow. The resulting code was multiple
times slower than GCC's implementation of gcov. Replace the fwrite/fread system
with an mmap() version.

If the `.gcda' file doesn't exist, we (re)allocate a buffer that we write
into. That gets written to the `.gcda' file in one chunk. If the `.gcda' file
already exists, we simply mmap() the file, modify the mapped data, and use
msync() to write the contents out to disk. It's much easier than implementing
our own buffering scheme, and we don't have to use fwrite's and fread's
buffering.

For those who are numbers-oriented, here are some timings:

GCC Verison
-----------

`.gcda' files don't exist:  23s
`.gcda' files do exist:     14s

LLVM Version (before this change)
---------------------------------

`.gcda' files don't exist:  28s
`.gcda' files do exist:     28s

LLVM Version (with this change)
-------------------------------

`.gcda' files don't exist:  18s
`.gcda' files do exist:      4s

It's a win-win-win-win-lose-win-win scenario!

<rdar://problem/13466086>


git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@182563 91177308-0d34-0410-b5e6-96231b3b80d8

11 files changed, 259 insertions, 66 deletions

diff --git a/SDKs/darwin/usr/include/fcntl.h b/SDKs/darwin/usr/include/fcntl.h
new file mode 100644
index 000000000..a5f91e3a5
--- /dev/null
+++ b/SDKs/darwin/usr/include/fcntl.h
@@ -0,0 +1,17 @@
+/* ===-- fcntl.h - stub SDK header for compiler-rt --------------------------===
+ *
+ *                     The LLVM Compiler Infrastructure
+ *
+ * This file is dual licensed under the MIT and the University of Illinois Open
+ * Source Licenses. See LICENSE.TXT for details.
+ *
+ * ===-----------------------------------------------------------------------===
+ *
+ * This is a stub SDK header file. This file is not part of the interface of
+ * this library nor an official version of the appropriate SDK header. It is
+ * intended only to stub the features of this header required by compiler-rt.
+ *
+ * ===-----------------------------------------------------------------------===
+ */
+
+#include <sys/fcntl.h>
diff --git a/SDKs/darwin/usr/include/stdio.h b/SDKs/darwin/usr/include/stdio.h
index 5bbd8a55b..006652fb9 100644
--- a/SDKs/darwin/usr/include/stdio.h
+++ b/SDKs/darwin/usr/include/stdio.h
@@ -24,15 +24,18 @@ extern "C" {
 typedef struct __sFILE FILE;
 typedef __SIZE_TYPE__ size_t;
 
-/* Determine the appropriate fopen() and fwrite() functions. */
+/* Determine the appropriate fdopen, fopen(), and fwrite() functions. */
 #if defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__)
 #  if defined(__i386)
+#    define __FDOPEN_NAME  "_fdopen$UNIX2003"
 #    define __FOPEN_NAME "_fopen$UNIX2003"
 #    define __FWRITE_NAME "_fwrite$UNIX2003"
 #  elif defined(__x86_64__)
+#    define __FDOPEN_NAME  "_fdopen"
 #    define __FOPEN_NAME "_fopen"
 #    define __FWRITE_NAME "_fwrite"
 #  elif defined(__arm)
+#    define __FDOPEN_NAME  "_fdopen"
 #    define __FOPEN_NAME "_fopen"
 #    define __FWRITE_NAME "_fwrite"
 #  else
@@ -40,9 +43,11 @@ typedef __SIZE_TYPE__ size_t;
 #  endif
 #elif defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__)
 #  if defined(__i386) || defined (__x86_64)
+#    define __FDOPEN_NAME  "_fdopen"
 #    define __FOPEN_NAME "_fopen"
 #    define __FWRITE_NAME "_fwrite"
 #  elif defined(__arm)
+#    define __FDOPEN_NAME  "_fdopen"
 #    define __FOPEN_NAME "_fopen"
 #    define __FWRITE_NAME "_fwrite"
 #  else
@@ -68,14 +73,13 @@ extern FILE *__stderrp;
 int fclose(FILE *);
 int fflush(FILE *);
 FILE *fopen(const char * __restrict, const char * __restrict) __asm(__FOPEN_NAME);
+FILE *fdopen(int, const char *) __asm(__FDOPEN_NAME);
 int fprintf(FILE * __restrict, const char * __restrict, ...);
 size_t fwrite(const void * __restrict, size_t, size_t, FILE * __restrict)
   __asm(__FWRITE_NAME);
 size_t fread(void * __restrict, size_t, size_t, FILE * __restrict);
 long ftell(FILE *);
 int fseek(FILE *, long, int);
-void setbuf(FILE * __restrict, char * __restrict);
-
 int snprintf(char * __restrict, size_t, const char * __restrict, ...);
 
 #if defined(__cplusplus)
diff --git a/SDKs/darwin/usr/include/stdlib.h b/SDKs/darwin/usr/include/stdlib.h
index 7c973dcc6..b6d3171cf 100644
--- a/SDKs/darwin/usr/include/stdlib.h
+++ b/SDKs/darwin/usr/include/stdlib.h
@@ -27,5 +27,6 @@ int atoi(const char *);
 void free(void *);
 char *getenv(const char *);
 void *malloc(size_t);
+void *realloc(void *, size_t);
 
 #endif /* __STDLIB_H__ */
diff --git a/SDKs/darwin/usr/include/sys/fcntl.h b/SDKs/darwin/usr/include/sys/fcntl.h
new file mode 100644
index 000000000..b71706bf4
--- /dev/null
+++ b/SDKs/darwin/usr/include/sys/fcntl.h
@@ -0,0 +1,52 @@
+/* ===-- fcntl.h - stub SDK header for compiler-rt --------------------------===
+ *
+ *                     The LLVM Compiler Infrastructure
+ *
+ * This file is dual licensed under the MIT and the University of Illinois Open
+ * Source Licenses. See LICENSE.TXT for details.
+ *
+ * ===-----------------------------------------------------------------------===
+ *
+ * This is a stub SDK header file. This file is not part of the interface of
+ * this library nor an official version of the appropriate SDK header. It is
+ * intended only to stub the features of this header required by compiler-rt.
+ *
+ * ===-----------------------------------------------------------------------===
+ */
+
+#ifndef _SYS_FCNTL_H_
+#define _SYS_FCNTL_H_
+
+/* Determine the appropriate open function. */
+#if defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__)
+#  if defined(__i386)
+#    define __OPEN_NAME  "_open$UNIX2003"
+#  elif defined(__x86_64__)
+#    define __OPEN_NAME  "_open"
+#  elif defined(__arm)
+#    define __OPEN_NAME  "_open"
+#  else
+#    error "unrecognized architecture for targetting OS X"
+#  endif
+#elif defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__)
+#  if defined(__i386) || defined (__x86_64)
+#    define __OPEN_NAME  "_open"
+#  elif defined(__arm)
+#    define __OPEN_NAME  "_open"
+#  else
+#    error "unrecognized architecture for targetting iOS"
+#  endif
+#else
+#  error "unrecognized architecture for targetting Darwin"
+#endif
+
+#define O_RDONLY   0x0000    /* open for reading only */
+#define O_WRONLY   0x0001    /* open for writing only */
+#define O_RDWR     0x0002    /* open for reading and writing */
+#define O_ACCMODE  0x0003    /* mask for above modes */
+
+#define O_CREAT    0x0200    /* create if nonexistant */
+
+int open(const char *, int, ...) __asm(__OPEN_NAME);
+
+#endif /* !_SYS_FCNTL_H_ */
diff --git a/SDKs/darwin/usr/include/sys/mman.h b/SDKs/darwin/usr/include/sys/mman.h
new file mode 100644
index 000000000..84561f1b6
--- /dev/null
+++ b/SDKs/darwin/usr/include/sys/mman.h
@@ -0,0 +1,42 @@
+/* ===-- mman.h - stub SDK header for compiler-rt ---------------------------===
+ *
+ *                     The LLVM Compiler Infrastructure
+ *
+ * This file is dual licensed under the MIT and the University of Illinois Open
+ * Source Licenses. See LICENSE.TXT for details.
+ *
+ * ===-----------------------------------------------------------------------===
+ *
+ * This is a stub SDK header file. This file is not part of the interface of
+ * this library nor an official version of the appropriate SDK header. It is
+ * intended only to stub the features of this header required by compiler-rt.
+ *
+ * ===-----------------------------------------------------------------------===
+ */
+
+#ifndef __SYS_MMAN_H__
+#define __SYS_MMAN_H__
+
+typedef __SIZE_TYPE__ size_t;
+
+#define PROT_NONE     0x00
+#define PROT_READ     0x01
+#define PROT_WRITE    0x02
+#define PROT_EXEC     0x04
+
+#define MAP_SHARED    0x0001
+#define MAP_PRIVATE   0x0002
+
+#define MAP_FILE      0x0000
+#define MAP_ANON      0x1000
+
+#define MS_ASYNC      0x0001
+#define MS_INVALIDATE 0x0002
+#define MS_SYNC       0x0010
+
+void *mmap(void *addr, size_t len, int prot, int flags, int fd,
+           long long offset);
+int munmap(void *addr, size_t len);
+int msync(void *addr, size_t len, int flags);
+
+#endif /* __SYS_MMAN_H__ */
diff --git a/SDKs/linux/usr/include/fcntl.h b/SDKs/linux/usr/include/fcntl.h
new file mode 100644
index 000000000..a5f91e3a5
--- /dev/null
+++ b/SDKs/linux/usr/include/fcntl.h
@@ -0,0 +1,17 @@
+/* ===-- fcntl.h - stub SDK header for compiler-rt --------------------------===
+ *
+ *                     The LLVM Compiler Infrastructure
+ *
+ * This file is dual licensed under the MIT and the University of Illinois Open
+ * Source Licenses. See LICENSE.TXT for details.
+ *
+ * ===-----------------------------------------------------------------------===
+ *
+ * This is a stub SDK header file. This file is not part of the interface of
+ * this library nor an official version of the appropriate SDK header. It is
+ * intended only to stub the features of this header required by compiler-rt.
+ *
+ * ===-----------------------------------------------------------------------===
+ */
+
+#include <sys/fcntl.h>
diff --git a/SDKs/linux/usr/include/stdio.h b/SDKs/linux/usr/include/stdio.h
index 0ca10fabe..fba593640 100644
--- a/SDKs/linux/usr/include/stdio.h
+++ b/SDKs/linux/usr/include/stdio.h
@@ -33,11 +33,11 @@ extern struct _IO_FILE *stderr;
 extern int fclose(FILE *);
 extern int fflush(FILE *);
 extern FILE *fopen(const char * restrict, const char * restrict);
+extern FILE *fdopen(int, const char * restrict);
 extern int fprintf(FILE * restrict, const char * restrict, ...);
 extern size_t fwrite(const void * restrict, size_t, size_t, FILE * restrict);
 extern size_t fread(void * restrict, size_t, size_t, FILE * restrict);
 extern long ftell(FILE *);
 extern int fseek(FILE *, long, int);
-extern void setbuf(FILE * restrict, char * restrict);
 
 #endif /* __STDIO_H__ */
diff --git a/SDKs/linux/usr/include/stdlib.h b/SDKs/linux/usr/include/stdlib.h
index e6b8bf710..966b29db6 100644
--- a/SDKs/linux/usr/include/stdlib.h
+++ b/SDKs/linux/usr/include/stdlib.h
@@ -30,5 +30,7 @@ char *getenv(const char *) __attribute__((__nothrow__))
   __attribute__((__warn_unused_result__));
 void *malloc(size_t) __attribute__((__nothrow__)) __attribute((__malloc__))
      __attribute__((__warn_unused_result__));
+void *realloc(void *, size_t) __attribute__((__nothrow__)) __attribute((__malloc__))
+     __attribute__((__warn_unused_result__));
 
 #endif /* __STDLIB_H__ */
diff --git a/SDKs/linux/usr/include/sys/fcntl.h b/SDKs/linux/usr/include/sys/fcntl.h
new file mode 100644
index 000000000..1512bf9b4
--- /dev/null
+++ b/SDKs/linux/usr/include/sys/fcntl.h
@@ -0,0 +1,29 @@
+/* ===-- fcntl.h - stub SDK header for compiler-rt --------------------------===
+ *
+ *                     The LLVM Compiler Infrastructure
+ *
+ * This file is dual licensed under the MIT and the University of Illinois Open
+ * Source Licenses. See LICENSE.TXT for details.
+ *
+ * ===-----------------------------------------------------------------------===
+ *
+ * This is a stub SDK header file. This file is not part of the interface of
+ * this library nor an official version of the appropriate SDK header. It is
+ * intended only to stub the features of this header required by compiler-rt.
+ *
+ * ===-----------------------------------------------------------------------===
+ */
+
+#ifndef _SYS_FCNTL_H_
+#define _SYS_FCNTL_H_
+
+#define O_RDONLY   0x0000
+#define O_WRONLY   0x0001
+#define O_RDWR     0x0002
+#define O_ACCMODE  0x0003
+
+#define O_CREAT    0x0200
+
+int open(const char *, int, ...);
+
+#endif /* _SYS_FCNTL_H_ */
diff --git a/SDKs/linux/usr/include/sys/mman.h b/SDKs/linux/usr/include/sys/mman.h
index 7c4d05181..bfb7f8bb0 100644
--- a/SDKs/linux/usr/include/sys/mman.h
+++ b/SDKs/linux/usr/include/sys/mman.h
@@ -19,10 +19,28 @@
 
 typedef __SIZE_TYPE__ size_t;
 
-#define PROT_READ 0x1
-#define PROT_WRITE 0x2
-#define PROT_EXEC 0x4
+#define PROT_NONE     0x00
+#define PROT_READ     0x01
+#define PROT_WRITE    0x02
+#define PROT_EXEC     0x04
 
+#define MAP_SHARED    0x0001
+#define MAP_PRIVATE   0x0002
+
+#define MAP_FILE      0x0000
+#define MAP_ANON      0x1000
+
+#define MS_ASYNC      0x0001
+#define MS_INVALIDATE 0x0002
+#define MS_SYNC       0x0010
+
+extern void *mmap(void *addr, size_t len, int prot, int flags, int fd,
+                  long long offset)
+  __attribute__((__nothrow__));
+extern int munmap(void *addr, size_t len)
+  __attribute__((__nothrow__));
+extern int msync(void *addr, size_t len, int flags)
+  __attribute__((__nothrow__));
 extern int mprotect (void *__addr, size_t __len, int __prot)
   __attribute__((__nothrow__));
 
diff --git a/lib/profile/GCDAProfiling.c b/lib/profile/GCDAProfiling.c
index 53ecff438..71dd9bd67 100644
--- a/lib/profile/GCDAProfiling.c
+++ b/lib/profile/GCDAProfiling.c
@@ -20,10 +20,12 @@
 |*
 \*===----------------------------------------------------------------------===*/
 
+#include <fcntl.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <sys/stat.h>
+#include <sys/mman.h>
 #include <sys/types.h>
 #ifdef _WIN32
 #include <direct.h>
@@ -50,11 +52,13 @@ static FILE *output_file = NULL;
 /*
  * Buffer that we write things into.
  */
-#define WRITE_BUFFER_SIZE (1 << 12)
-static char write_buffer[WRITE_BUFFER_SIZE];
-static int64_t cur_file_pos = 0;
-static uint32_t cur_offset = 0;
+#define NEW_WRITE_BUFFER_SIZE (128 * 1024)
+static char *new_write_buffer = NULL;
+static uint64_t cur_buffer_size = 0;
+static uint64_t cur_pos = 0;
+static uint64_t file_size = 0;
 static int new_file = 0;
+static int fd = -1;
 
 /*
  * A list of functions to write out the data.
@@ -82,30 +86,20 @@ struct flush_fn_node {
 static struct flush_fn_node *flush_fn_head = NULL;
 static struct flush_fn_node *flush_fn_tail = NULL;
 
-static void flush_buffer() {
-  /* Flush the buffer to file. */
-  fwrite(write_buffer, cur_offset, 1, output_file);
-
-  if (new_file) {
-    cur_offset = 0;
-    return;
-  }
-
-  /* Read in more of the file. */
-  fread(write_buffer,1, WRITE_BUFFER_SIZE, output_file);
-  fseek(output_file, cur_file_pos, SEEK_SET);
-
-  cur_offset = 0;
+static void resize_write_buffer(uint64_t size) {
+  if (!new_file) return;
+  size += cur_pos;
+  if (size <= cur_buffer_size) return;
+  size = (size - 1) / NEW_WRITE_BUFFER_SIZE + 1;
+  size *= NEW_WRITE_BUFFER_SIZE;
+  new_write_buffer = realloc(new_write_buffer, size);
+  cur_buffer_size = size;
 }
 
 static void write_bytes(const char *s, size_t len) {
-  if (cur_offset + len > WRITE_BUFFER_SIZE)
-    flush_buffer();
-
-  cur_file_pos += len;
-
-  for (; len > 0; --len)
-    write_buffer[cur_offset++] = *s++;
+  resize_write_buffer(len);
+  memcpy(&new_write_buffer[cur_pos], s, len);
+  cur_pos += len;
 }
 
 static void write_32bit_value(uint32_t i) {
@@ -130,30 +124,22 @@ static void write_string(const char *s) {
 static uint32_t read_32bit_value() {
   uint32_t val;
 
-  if (cur_offset + 4 > WRITE_BUFFER_SIZE)
-    flush_buffer();
-
   if (new_file)
     return (uint32_t)-1;
 
-  val = *(uint32_t*)&write_buffer[cur_offset];
-  cur_file_pos += 4;
-  cur_offset += 4;
+  val = *(uint32_t*)&new_write_buffer[cur_pos];
+  cur_pos += 4;
   return val;
 }
 
 static uint64_t read_64bit_value() {
   uint64_t val;
 
-  if (cur_offset + 8 > WRITE_BUFFER_SIZE)
-    flush_buffer();
-
   if (new_file)
-    return (uint32_t)-1;
+    return (uint64_t)-1;
 
-  val = *(uint64_t*)&write_buffer[cur_offset];
-  cur_file_pos += 8;
-  cur_offset += 8;
+  val = *(uint64_t*)&new_write_buffer[cur_pos];
+  cur_pos += 8;
   return val;
 }
 
@@ -210,6 +196,21 @@ static void recursive_mkdir(char *filename) {
   }
 }
 
+static void map_file() {
+  fseek(output_file, 0L, SEEK_END);
+  file_size = ftell(output_file);
+
+  new_write_buffer = mmap(0, file_size, PROT_READ | PROT_WRITE,
+                          MAP_FILE | MAP_SHARED, fd, 0);
+}
+
+static void unmap_file() {
+  msync(new_write_buffer, file_size, MS_SYNC);
+  munmap(new_write_buffer, file_size);
+  new_write_buffer = NULL;
+  file_size = 0;
+}
+
 /*
  * --- LLVM line counter API ---
  */
@@ -220,18 +221,21 @@ static void recursive_mkdir(char *filename) {
  */
 void llvm_gcda_start_file(const char *orig_filename, const char version[4]) {
   char *filename = mangle_filename(orig_filename);
+  const char *mode = "r+b";
 
   /* Try just opening the file. */
-  output_file = fopen(filename, "r+b");
+  new_file = 0;
+  fd = open(filename, O_RDWR);
 
-  if (!output_file) {
+  if (fd == -1) {
     /* Try opening the file, creating it if necessary. */
     new_file = 1;
-    output_file = fopen(filename, "w+b");
-    if (!output_file) {
+    mode = "w+b";
+    fd = open(filename, O_RDWR | O_CREAT);
+    if (fd == -1) {
       /* Try creating the directories first then opening the file. */
       recursive_mkdir(filename);
-      output_file = fopen(filename, "w+b");
+      fd = open(filename, O_RDWR | O_CREAT);
       if (!output_file) {
         /* Bah! It's hopeless. */
         fprintf(stderr, "profiling:%s: cannot open\n", filename);
@@ -241,14 +245,19 @@ void llvm_gcda_start_file(const char *orig_filename, const char version[4]) {
     }
   }
 
-  setbuf(output_file, 0);
+  output_file = fdopen(fd, mode);
 
   /* Initialize the write buffer. */
-  memset(write_buffer, 0, WRITE_BUFFER_SIZE);
-  fread(write_buffer, 1, WRITE_BUFFER_SIZE, output_file);
-  fseek(output_file, 0L, SEEK_SET);
-  cur_file_pos = 0;
-  cur_offset = 0;
+  new_write_buffer = NULL;
+  cur_buffer_size = 0;
+  cur_pos = 0;
+
+  if (new_file) {
+    resize_write_buffer(NEW_WRITE_BUFFER_SIZE);
+    memset(new_write_buffer, 0, NEW_WRITE_BUFFER_SIZE);
+  } else {
+    map_file();
+  }
 
   /* gcda file, version, stamp LLVM. */
   write_bytes("adcg", 4);
@@ -315,12 +324,11 @@ void llvm_gcda_emit_function(uint32_t ident, const char *function_name,
 void llvm_gcda_emit_arcs(uint32_t num_counters, uint64_t *counters) {
   uint32_t i;
   uint64_t *old_ctrs = NULL;
-  int64_t save_cur_file_pos = cur_file_pos;
   uint32_t val = 0;
+  uint64_t save_cur_pos = cur_pos;
 
   if (!output_file) return;
 
-  flush_buffer();
   val = read_32bit_value();
 
   if (val != (uint32_t)-1) {
@@ -339,15 +347,10 @@ void llvm_gcda_emit_arcs(uint32_t num_counters, uint64_t *counters) {
     old_ctrs = malloc(sizeof(uint64_t) * num_counters);
     for (i = 0; i < num_counters; ++i)
       old_ctrs[i] = read_64bit_value();
-
-    /* Reset the current buffer and file position. */
-    memset(write_buffer, 0, WRITE_BUFFER_SIZE);
-    cur_file_pos = save_cur_file_pos;
-    cur_offset = 0;
-
-    fseek(output_file, cur_file_pos, SEEK_SET);
   }
 
+  cur_pos = save_cur_pos;
+
   /* Counter #1 (arcs) tag */
   write_bytes("\0\0\xa1\1", 4);
   write_32bit_value(num_counters * 2);
@@ -369,9 +372,17 @@ void llvm_gcda_end_file() {
   /* Write out EOF record. */
   if (!output_file) return;
   write_bytes("\0\0\0\0\0\0\0\0", 8);
-  flush_buffer();
+
+  if (new_file) {
+    fwrite(new_write_buffer, cur_pos, 1, output_file);
+    free(new_write_buffer);
+  } else {
+    unmap_file();
+  }
+
   fclose(output_file);
   output_file = NULL;
+  new_write_buffer = NULL;
 
 #ifdef DEBUG_GCDAPROFILING
   fprintf(stderr, "llvmgcda: -----\n");