[msan] Fix return type of mbrtowc

Summary: Fixes https://github.com/google/oss-fuzz/issues/1009

Reviewers: eugenis

Subscribers: llvm-commits

Differential Revision: https://reviews.llvm.org/D40676

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@319484 91177308-0d34-0410-b5e6-96231b3b80d8

2 files changed, 11 insertions, 8 deletions

diff --git a/lib/msan/msan_interceptors.cc b/lib/msan/msan_interceptors.cc
index 5abfb9ad0..0b58f54b5 100644
--- a/lib/msan/msan_interceptors.cc
+++ b/lib/msan/msan_interceptors.cc
@@ -526,7 +526,8 @@ INTERCEPTOR(int, mbtowc, wchar_t *dest, const char *src, SIZE_T n) {
   return res;
 }
 
-INTERCEPTOR(int, mbrtowc, wchar_t *dest, const char *src, SIZE_T n, void *ps) {
+INTERCEPTOR(SIZE_T, mbrtowc, wchar_t *dest, const char *src, SIZE_T n,
+            void *ps) {
   ENSURE_MSAN_INITED();
   SIZE_T res = REAL(mbrtowc)(dest, src, n, ps);
   if (res != (SIZE_T)-1 && dest) __msan_unpoison(dest, sizeof(wchar_t));
diff --git a/lib/msan/tests/msan_test.cc b/lib/msan/tests/msan_test.cc
index 212e0ee3e..82d659e57 100644
--- a/lib/msan/tests/msan_test.cc
+++ b/lib/msan/tests/msan_test.cc
@@ -2121,13 +2121,15 @@ TEST(MemorySanitizer, mbtowc) {
 }
 
 TEST(MemorySanitizer, mbrtowc) {
-  const char *x = "abc";
-  wchar_t wx;
-  mbstate_t mbs;
-  memset(&mbs, 0, sizeof(mbs));
-  int res = mbrtowc(&wx, x, 3, &mbs);
-  EXPECT_GT(res, 0);
-  EXPECT_NOT_POISONED(wx);
+  mbstate_t mbs = {};
+
+  wchar_t wc;
+  size_t res = mbrtowc(&wc, "\377", 1, &mbs);
+  EXPECT_EQ(res, -1ULL);
+
+  res = mbrtowc(&wc, "abc", 3, &mbs);
+  EXPECT_GT(res, 0ULL);
+  EXPECT_NOT_POISONED(wc);
 }
 
 TEST(MemorySanitizer, wcsftime) {