Outline several fbstring/malloc functions

Summary: Outline all the non-trivial constructor and the destructor, `assign()`, and `append()` to reduce code size. The fast path of copy, and moves, are instead still inlined.

Reviewed By: philippv

Differential Revision: D3896612

fbshipit-source-id: 25050d4ba28d25da226a7bc49d5b542947d0c512

diff --git a/folly/FBString.h b/folly/FBString.h
index 6730d1157df9ceefccd379ec2fc5088d4767bc49..e744e1f3bf4bf16780ce0d466af9f304b6b8261b 100644 (file)
--- a/folly/FBString.h
+++ b/folly/FBString.h
@@ -362,15 +362,10 @@ public:
   }
 
   ~fbstring_core() noexcept {
-    auto const c = category();
-    if (c == Category::isSmall) {
-      return;
-    }
-    if (c == Category::isMedium) {
-      free(ml_.data_);
+    if (category() == Category::isSmall) {
       return;
     }
-    RefCounted::decrementRefs(ml_.data_);
+    destroyMediumLarge();
   }
 
   // Snatches a previously mallocated string. The parameter "size"
@@ -444,6 +439,7 @@ public:
     }
   }
 
+  FOLLY_MALLOC_NOINLINE
   void reserve(size_t minCapacity, bool disableSSO = FBSTRING_DISABLE_SSO) {
     switch (category()) {
       case Category::isSmall:
@@ -514,6 +510,16 @@ private:
     setSmallSize(0);
   }
 
+  FOLLY_MALLOC_NOINLINE void destroyMediumLarge() noexcept {
+    auto const c = category();
+    FBSTRING_ASSERT(c != Category::isSmall);
+    if (c == Category::isMedium) {
+      free(ml_.data_);
+    } else {
+      RefCounted::decrementRefs(ml_.data_);
+    }
+  }
+
   struct RefCounted {
     std::atomic<size_t> refCount_;
     Char data_[1];
@@ -665,6 +671,7 @@ private:
   void shrinkMedium(size_t delta);
   void shrinkLarge(size_t delta);
 
+  void unshare(size_t minCapacity = 0);
   Char* mutableDataLarge();
 };
 
@@ -688,7 +695,8 @@ inline void fbstring_core<Char>::copySmall(const fbstring_core& rhs) {
 }
 
 template <class Char>
-inline void fbstring_core<Char>::copyMedium(const fbstring_core& rhs) {
+FOLLY_MALLOC_NOINLINE inline void fbstring_core<Char>::copyMedium(
+    const fbstring_core& rhs) {
   // Medium strings are copied eagerly. Don't forget to allocate
   // one extra Char for the null terminator.
   auto const allocSize = goodMallocSize((1 + rhs.ml_.size_) * sizeof(Char));
@@ -702,7 +710,8 @@ inline void fbstring_core<Char>::copyMedium(const fbstring_core& rhs) {
 }
 
 template <class Char>
-inline void fbstring_core<Char>::copyLarge(const fbstring_core& rhs) {
+FOLLY_MALLOC_NOINLINE inline void fbstring_core<Char>::copyLarge(
+    const fbstring_core& rhs) {
   // Large strings are just refcounted
   ml_ = rhs.ml_;
   RefCounted::incrementRefs(ml_.data_);
@@ -754,7 +763,7 @@ inline void fbstring_core<Char>::initSmall(
 }
 
 template <class Char>
-inline void fbstring_core<Char>::initMedium(
+FOLLY_MALLOC_NOINLINE inline void fbstring_core<Char>::initMedium(
     const Char* const data, const size_t size) {
   // Medium strings are allocated normally. Don't forget to
   // allocate one extra Char for the terminating null.
@@ -767,7 +776,7 @@ inline void fbstring_core<Char>::initMedium(
 }
 
 template <class Char>
-inline void fbstring_core<Char>::initLarge(
+FOLLY_MALLOC_NOINLINE inline void fbstring_core<Char>::initLarge(
     const Char* const data, const size_t size) {
   // Large strings are allocated differently
   size_t effectiveCapacity = size;
@@ -778,44 +787,43 @@ inline void fbstring_core<Char>::initLarge(
   ml_.data_[size] = '\0';
 }
 
+template <class Char>
+FOLLY_MALLOC_NOINLINE inline void fbstring_core<Char>::unshare(
+    size_t minCapacity) {
+  FBSTRING_ASSERT(category() == Category::isLarge);
+  size_t effectiveCapacity = std::max(minCapacity, ml_.capacity());
+  auto const newRC = RefCounted::create(&effectiveCapacity);
+  // If this fails, someone placed the wrong capacity in an
+  // fbstring.
+  FBSTRING_ASSERT(effectiveCapacity >= ml_.capacity());
+  // Also copies terminator.
+  fbstring_detail::podCopy(ml_.data_, ml_.data_ + ml_.size_ + 1, newRC->data_);
+  RefCounted::decrementRefs(ml_.data_);
+  ml_.data_ = newRC->data_;
+  ml_.setCapacity(effectiveCapacity, Category::isLarge);
+  // size_ remains unchanged.
+}
+
 template <class Char>
 inline Char* fbstring_core<Char>::mutableDataLarge() {
   FBSTRING_ASSERT(category() == Category::isLarge);
-  if (RefCounted::refs(ml_.data_) > 1) {
-    // Ensure unique.
-    size_t effectiveCapacity = ml_.capacity();
-    auto const newRC = RefCounted::create(&effectiveCapacity);
-    // If this fails, someone placed the wrong capacity in an
-    // fbstring.
-    FBSTRING_ASSERT(effectiveCapacity >= ml_.capacity());
-    // Also copies terminator.
-    fbstring_detail::podCopy(
-        ml_.data_, ml_.data_ + ml_.size_ + 1, newRC->data_);
-    RefCounted::decrementRefs(ml_.data_);
-    ml_.data_ = newRC->data_;
+  if (RefCounted::refs(ml_.data_) > 1) { // Ensure unique.
+    unshare();
   }
   return ml_.data_;
 }
 
 template <class Char>
-inline void fbstring_core<Char>::reserveLarge(size_t minCapacity) {
+FOLLY_MALLOC_NOINLINE inline void fbstring_core<Char>::reserveLarge(
+    size_t minCapacity) {
   FBSTRING_ASSERT(category() == Category::isLarge);
-  // Ensure unique
-  if (RefCounted::refs(ml_.data_) > 1) {
+  if (RefCounted::refs(ml_.data_) > 1) { // Ensure unique
     // We must make it unique regardless; in-place reallocation is
     // useless if the string is shared. In order to not surprise
     // people, reserve the new block at current capacity or
     // more. That way, a string's capacity never shrinks after a
     // call to reserve.
-    minCapacity = std::max(minCapacity, ml_.capacity());
-    auto const newRC = RefCounted::create(&minCapacity);
-    // Also copies terminator.
-    fbstring_detail::podCopy(
-        ml_.data_, ml_.data_ + ml_.size_ + 1, newRC->data_);
-    RefCounted::decrementRefs(ml_.data_);
-    ml_.data_ = newRC->data_;
-    ml_.setCapacity(minCapacity, Category::isLarge);
-    // size remains unchanged
+    unshare(minCapacity);
   } else {
     // String is not shared, so let's try to realloc (if needed)
     if (minCapacity > ml_.capacity()) {
@@ -830,7 +838,8 @@ inline void fbstring_core<Char>::reserveLarge(size_t minCapacity) {
 }
 
 template <class Char>
-inline void fbstring_core<Char>::reserveMedium(const size_t minCapacity) {
+FOLLY_MALLOC_NOINLINE inline void fbstring_core<Char>::reserveMedium(
+    const size_t minCapacity) {
   FBSTRING_ASSERT(category() == Category::isMedium);
   // String is not shared
   if (minCapacity <= ml_.capacity()) {
@@ -862,7 +871,7 @@ inline void fbstring_core<Char>::reserveMedium(const size_t minCapacity) {
 }
 
 template <class Char>
-inline void fbstring_core<Char>::reserveSmall(
+FOLLY_MALLOC_NOINLINE inline void fbstring_core<Char>::reserveSmall(
     size_t minCapacity, const bool disableSSO) {
   FBSTRING_ASSERT(category() == Category::isSmall);
   if (!disableSSO && minCapacity <= maxSmallSize) {
@@ -1140,21 +1149,23 @@ public:
     assign(str, pos, n);
   }
 
+  FOLLY_MALLOC_NOINLINE
   /* implicit */ basic_fbstring(const value_type* s, const A& /*a*/ = A())
-      : store_(s, basic_fbstring::traitsLength(s)) {
-  }
+      : store_(s, traitsLength(s)) {}
 
+  FOLLY_MALLOC_NOINLINE
   basic_fbstring(const value_type* s, size_type n, const A& /*a*/ = A())
       : store_(s, n) {
   }
 
+  FOLLY_MALLOC_NOINLINE
   basic_fbstring(size_type n, value_type c, const A& /*a*/ = A()) {
     auto const pData = store_.expandNoinit(n);
     fbstring_detail::podFill(pData, pData + n, c);
   }
 
   template <class InIt>
-  basic_fbstring(
+  FOLLY_MALLOC_NOINLINE basic_fbstring(
       InIt begin,
       InIt end,
       typename std::enable_if<
@@ -1164,6 +1175,7 @@ public:
   }
 
   // Specialization for const char*, const char*
+  FOLLY_MALLOC_NOINLINE
   basic_fbstring(const value_type* b, const value_type* e, const A& /*a*/ = A())
       : store_(b, e - b) {
   }
@@ -1175,12 +1187,12 @@ public:
   }
 
   // Construction from initialization list
+  FOLLY_MALLOC_NOINLINE
   basic_fbstring(std::initializer_list<value_type> il) {
     assign(il.begin(), il.end());
   }
 
-  ~basic_fbstring() noexcept {
-  }
+  ~basic_fbstring() noexcept {}
 
   basic_fbstring& operator=(const basic_fbstring& lhs);
 
@@ -1352,7 +1364,7 @@ public:
   basic_fbstring& append(const value_type* s, size_type n);
 
   basic_fbstring& append(const value_type* s) {
-    return append(s, traits_type::length(s));
+    return append(s, traitsLength(s));
   }
 
   basic_fbstring& append(size_type n, value_type c);
@@ -1386,7 +1398,7 @@ public:
   basic_fbstring& assign(const value_type* s, const size_type n);
 
   basic_fbstring& assign(const value_type* s) {
-    return assign(s, traits_type::length(s));
+    return assign(s, traitsLength(s));
   }
 
   basic_fbstring& assign(std::initializer_list<value_type> il) {
@@ -1416,7 +1428,7 @@ public:
   }
 
   basic_fbstring& insert(size_type pos, const value_type* s) {
-    return insert(pos, s, traits_type::length(s));
+    return insert(pos, s, traitsLength(s));
   }
 
   basic_fbstring& insert(size_type pos, size_type n, value_type c) {
@@ -1525,7 +1537,7 @@ public:
 
   // Replaces at most n1 chars of *this, starting with pos, with chars from s
   basic_fbstring& replace(size_type pos, size_type n1, const value_type* s) {
-    return replace(pos, n1, s, traits_type::length(s));
+    return replace(pos, n1, s, traitsLength(s));
   }
 
   // Replaces at most n1 chars of *this, starting with pos, with n2
@@ -1551,7 +1563,7 @@ public:
   }
 
   basic_fbstring& replace(iterator i1, iterator i2, const value_type* s) {
-    return replace(i1, i2, s, traits_type::length(s));
+    return replace(i1, i2, s, traitsLength(s));
   }
 
 private:
@@ -1654,7 +1666,7 @@ private:
       const;
 
   size_type find(const value_type* s, size_type pos = 0) const {
-    return find(s, pos, traits_type::length(s));
+    return find(s, pos, traitsLength(s));
   }
 
   size_type find (value_type c, size_type pos = 0) const {
@@ -1668,7 +1680,7 @@ private:
   size_type rfind(const value_type* s, size_type pos, size_type n) const;
 
   size_type rfind(const value_type* s, size_type pos = npos) const {
-    return rfind(s, pos, traits_type::length(s));
+    return rfind(s, pos, traitsLength(s));
   }
 
   size_type rfind(value_type c, size_type pos = npos) const {
@@ -1683,7 +1695,7 @@ private:
       const;
 
   size_type find_first_of(const value_type* s, size_type pos = 0) const {
-    return find_first_of(s, pos, traits_type::length(s));
+    return find_first_of(s, pos, traitsLength(s));
   }
 
   size_type find_first_of(value_type c, size_type pos = 0) const {
@@ -1699,7 +1711,7 @@ private:
 
   size_type find_last_of (const value_type* s,
                           size_type pos = npos) const {
-    return find_last_of(s, pos, traits_type::length(s));
+    return find_last_of(s, pos, traitsLength(s));
   }
 
   size_type find_last_of (value_type c, size_type pos = npos) const {
@@ -1716,7 +1728,7 @@ private:
 
   size_type find_first_not_of(const value_type* s,
                               size_type pos = 0) const {
-    return find_first_not_of(s, pos, traits_type::length(s));
+    return find_first_not_of(s, pos, traitsLength(s));
   }
 
   size_type find_first_not_of(value_type c, size_type pos = 0) const {
@@ -1733,7 +1745,7 @@ private:
 
   size_type find_last_not_of(const value_type* s,
                              size_type pos = npos) const {
-    return find_last_not_of(s, pos, traits_type::length(s));
+    return find_last_not_of(s, pos, traitsLength(s));
   }
 
   size_type find_last_not_of (value_type c, size_type pos = npos) const {
@@ -1766,7 +1778,7 @@ private:
 
   int compare(size_type pos1, size_type n1,
               const value_type* s) const {
-    return compare(pos1, n1, s, traits_type::length(s));
+    return compare(pos1, n1, s, traitsLength(s));
   }
 
   int compare(size_type pos1, size_type n1,
@@ -1788,9 +1800,9 @@ private:
 
   // Code from Jean-Francois Bastien (03/26/2007)
   int compare(const value_type* s) const {
-    // Could forward to compare(0, size(), s, traits_type::length(s))
+    // Could forward to compare(0, size(), s, traitsLength(s))
     // but that does two extra checks
-    const size_type n1(size()), n2(traits_type::length(s));
+    const size_type n1(size()), n2(traitsLength(s));
     const int r = traits_type::compare(data(), s, std::min(n1, n2));
     return r != 0 ? r : n1 > n2 ? 1 : n1 < n2 ? -1 : 0;
   }
@@ -1801,7 +1813,7 @@ private:
 };
 
 template <typename E, class T, class A, class S>
-inline typename basic_fbstring<E, T, A, S>::size_type
+FOLLY_MALLOC_NOINLINE inline typename basic_fbstring<E, T, A, S>::size_type
 basic_fbstring<E, T, A, S>::traitsLength(const value_type* s) {
   return s ? traits_type::length(s)
            : (std::__throw_logic_error(
@@ -1891,8 +1903,8 @@ inline basic_fbstring<E, T, A, S>& basic_fbstring<E, T, A, S>::append(
 }
 
 template <typename E, class T, class A, class S>
-inline basic_fbstring<E, T, A, S>& basic_fbstring<E, T, A, S>::append(
-    const value_type* s, size_type n) {
+FOLLY_MALLOC_NOINLINE inline basic_fbstring<E, T, A, S>&
+basic_fbstring<E, T, A, S>::append(const value_type* s, size_type n) {
   Invariant checker(*this);
 
   if (FBSTRING_UNLIKELY(!n)) {
@@ -1942,11 +1954,10 @@ inline basic_fbstring<E, T, A, S>& basic_fbstring<E, T, A, S>::assign(
 }
 
 template <typename E, class T, class A, class S>
-inline basic_fbstring<E, T, A, S>& basic_fbstring<E, T, A, S>::assign(
-    const value_type* s, const size_type n) {
+FOLLY_MALLOC_NOINLINE inline basic_fbstring<E, T, A, S>&
+basic_fbstring<E, T, A, S>::assign(const value_type* s, const size_type n) {
   Invariant checker(*this);
 
-  // s can alias this, we need to use podMove.
   if (n == 0) {
     resize(0);
   } else if (size() >= n) {

diff --git a/folly/Malloc.h b/folly/Malloc.h
index 8092ab0b197772df2397bad9c514541ab49fe8f5..de75e45195b573db2bdb447a384b384f279c6e7b 100644 (file)
--- a/folly/Malloc.h
+++ b/folly/Malloc.h
@@ -111,6 +111,7 @@ __attribute__((__weak__));
 #include <cstdlib>
 #include <cstring>
 
+#include <atomic>
 #include <new>
 
 #ifdef _LIBSTDCXX_FBSTRING
@@ -121,17 +122,21 @@ namespace folly {
 #endif
 
 // Cannot depend on Portability.h when _LIBSTDCXX_FBSTRING.
-// Disabled for nvcc because it fails on attributes on lambdas.
-#if defined(__GNUC__) && !defined(__NVCC__)
+#if defined(__GNUC__)
 #define FOLLY_MALLOC_NOINLINE __attribute__((__noinline__))
+// This is for checked malloc-like functions (returns non-null pointer
+// which cannot alias any outstanding pointer).
+#define FOLLY_MALLOC_CHECKED_MALLOC                     \
+  __attribute__((__returns_nonnull__, __malloc__))
 #else
 #define FOLLY_MALLOC_NOINLINE
+#define FOLLY_MALLOC_CHECKED_MALLOC
 #endif
 
 /**
  * Determine if we are using jemalloc or not.
  */
-inline bool usingJEMalloc() noexcept {
+FOLLY_MALLOC_NOINLINE inline bool usingJEMalloc() noexcept {
   // Checking for rallocx != NULL is not sufficient; we may be in a dlopen()ed
   // module that depends on libjemalloc, so rallocx is resolved, but the main
   // program might be using a different memory allocator.
@@ -140,7 +145,7 @@ inline bool usingJEMalloc() noexcept {
   // per-thread counter of allocated memory increases. This makes me
   // feel dirty inside. Also note that this requires jemalloc to have
   // been compiled with --enable-stats.
-  static const bool result = [] () FOLLY_MALLOC_NOINLINE noexcept {
+  static const bool result = [] () noexcept {
     // Some platforms (*cough* OSX *cough*) require weak symbol checks to be
     // in the form if (mallctl != nullptr). Not if (mallctl) or if (!mallctl)
     // (!!). http://goo.gl/xpmctm
@@ -235,10 +240,11 @@ inline void* checkedRealloc(void* ptr, size_t size) {
  * routine just tries to call realloc() (thus benefitting of potential
  * copy-free coalescing) unless there's too much slack memory.
  */
-inline void* smartRealloc(void* p,
-                          const size_t currentSize,
-                          const size_t currentCapacity,
-                          const size_t newCapacity) {
+FOLLY_MALLOC_CHECKED_MALLOC FOLLY_MALLOC_NOINLINE inline void* smartRealloc(
+    void* p,
+    const size_t currentSize,
+    const size_t currentCapacity,
+    const size_t newCapacity) {
   assert(p);
   assert(currentSize <= currentCapacity &&
          currentCapacity < newCapacity);