From 2ff9d1814ee6477181ae333f6a926f290a8c5a96 Mon Sep 17 00:00:00 2001 From: Giuseppe Ottaviano Date: Mon, 3 Oct 2016 21:39:06 -0700 Subject: [PATCH] Outline several fbstring/malloc functions Summary: Outline all the non-trivial constructor and the destructor, `assign()`, and `append()` to reduce code size. The fast path of copy, and moves, are instead still inlined. Reviewed By: philippv Differential Revision: D3896612 fbshipit-source-id: 25050d4ba28d25da226a7bc49d5b542947d0c512 --- folly/FBString.h | 135 +++++++++++++++++++++++++---------------------- folly/Malloc.h | 22 +++++--- 2 files changed, 87 insertions(+), 70 deletions(-) diff --git a/folly/FBString.h b/folly/FBString.h index 6730d115..e744e1f3 100644 --- a/folly/FBString.h +++ b/folly/FBString.h @@ -362,15 +362,10 @@ public: } ~fbstring_core() noexcept { - auto const c = category(); - if (c == Category::isSmall) { - return; - } - if (c == Category::isMedium) { - free(ml_.data_); + if (category() == Category::isSmall) { return; } - RefCounted::decrementRefs(ml_.data_); + destroyMediumLarge(); } // Snatches a previously mallocated string. The parameter "size" @@ -444,6 +439,7 @@ public: } } + FOLLY_MALLOC_NOINLINE void reserve(size_t minCapacity, bool disableSSO = FBSTRING_DISABLE_SSO) { switch (category()) { case Category::isSmall: @@ -514,6 +510,16 @@ private: setSmallSize(0); } + FOLLY_MALLOC_NOINLINE void destroyMediumLarge() noexcept { + auto const c = category(); + FBSTRING_ASSERT(c != Category::isSmall); + if (c == Category::isMedium) { + free(ml_.data_); + } else { + RefCounted::decrementRefs(ml_.data_); + } + } + struct RefCounted { std::atomic refCount_; Char data_[1]; @@ -665,6 +671,7 @@ private: void shrinkMedium(size_t delta); void shrinkLarge(size_t delta); + void unshare(size_t minCapacity = 0); Char* mutableDataLarge(); }; @@ -688,7 +695,8 @@ inline void fbstring_core::copySmall(const fbstring_core& rhs) { } template -inline void fbstring_core::copyMedium(const fbstring_core& rhs) { +FOLLY_MALLOC_NOINLINE inline void fbstring_core::copyMedium( + const fbstring_core& rhs) { // Medium strings are copied eagerly. Don't forget to allocate // one extra Char for the null terminator. auto const allocSize = goodMallocSize((1 + rhs.ml_.size_) * sizeof(Char)); @@ -702,7 +710,8 @@ inline void fbstring_core::copyMedium(const fbstring_core& rhs) { } template -inline void fbstring_core::copyLarge(const fbstring_core& rhs) { +FOLLY_MALLOC_NOINLINE inline void fbstring_core::copyLarge( + const fbstring_core& rhs) { // Large strings are just refcounted ml_ = rhs.ml_; RefCounted::incrementRefs(ml_.data_); @@ -754,7 +763,7 @@ inline void fbstring_core::initSmall( } template -inline void fbstring_core::initMedium( +FOLLY_MALLOC_NOINLINE inline void fbstring_core::initMedium( const Char* const data, const size_t size) { // Medium strings are allocated normally. Don't forget to // allocate one extra Char for the terminating null. @@ -767,7 +776,7 @@ inline void fbstring_core::initMedium( } template -inline void fbstring_core::initLarge( +FOLLY_MALLOC_NOINLINE inline void fbstring_core::initLarge( const Char* const data, const size_t size) { // Large strings are allocated differently size_t effectiveCapacity = size; @@ -778,44 +787,43 @@ inline void fbstring_core::initLarge( ml_.data_[size] = '\0'; } +template +FOLLY_MALLOC_NOINLINE inline void fbstring_core::unshare( + size_t minCapacity) { + FBSTRING_ASSERT(category() == Category::isLarge); + size_t effectiveCapacity = std::max(minCapacity, ml_.capacity()); + auto const newRC = RefCounted::create(&effectiveCapacity); + // If this fails, someone placed the wrong capacity in an + // fbstring. + FBSTRING_ASSERT(effectiveCapacity >= ml_.capacity()); + // Also copies terminator. + fbstring_detail::podCopy(ml_.data_, ml_.data_ + ml_.size_ + 1, newRC->data_); + RefCounted::decrementRefs(ml_.data_); + ml_.data_ = newRC->data_; + ml_.setCapacity(effectiveCapacity, Category::isLarge); + // size_ remains unchanged. +} + template inline Char* fbstring_core::mutableDataLarge() { FBSTRING_ASSERT(category() == Category::isLarge); - if (RefCounted::refs(ml_.data_) > 1) { - // Ensure unique. - size_t effectiveCapacity = ml_.capacity(); - auto const newRC = RefCounted::create(&effectiveCapacity); - // If this fails, someone placed the wrong capacity in an - // fbstring. - FBSTRING_ASSERT(effectiveCapacity >= ml_.capacity()); - // Also copies terminator. - fbstring_detail::podCopy( - ml_.data_, ml_.data_ + ml_.size_ + 1, newRC->data_); - RefCounted::decrementRefs(ml_.data_); - ml_.data_ = newRC->data_; + if (RefCounted::refs(ml_.data_) > 1) { // Ensure unique. + unshare(); } return ml_.data_; } template -inline void fbstring_core::reserveLarge(size_t minCapacity) { +FOLLY_MALLOC_NOINLINE inline void fbstring_core::reserveLarge( + size_t minCapacity) { FBSTRING_ASSERT(category() == Category::isLarge); - // Ensure unique - if (RefCounted::refs(ml_.data_) > 1) { + if (RefCounted::refs(ml_.data_) > 1) { // Ensure unique // We must make it unique regardless; in-place reallocation is // useless if the string is shared. In order to not surprise // people, reserve the new block at current capacity or // more. That way, a string's capacity never shrinks after a // call to reserve. - minCapacity = std::max(minCapacity, ml_.capacity()); - auto const newRC = RefCounted::create(&minCapacity); - // Also copies terminator. - fbstring_detail::podCopy( - ml_.data_, ml_.data_ + ml_.size_ + 1, newRC->data_); - RefCounted::decrementRefs(ml_.data_); - ml_.data_ = newRC->data_; - ml_.setCapacity(minCapacity, Category::isLarge); - // size remains unchanged + unshare(minCapacity); } else { // String is not shared, so let's try to realloc (if needed) if (minCapacity > ml_.capacity()) { @@ -830,7 +838,8 @@ inline void fbstring_core::reserveLarge(size_t minCapacity) { } template -inline void fbstring_core::reserveMedium(const size_t minCapacity) { +FOLLY_MALLOC_NOINLINE inline void fbstring_core::reserveMedium( + const size_t minCapacity) { FBSTRING_ASSERT(category() == Category::isMedium); // String is not shared if (minCapacity <= ml_.capacity()) { @@ -862,7 +871,7 @@ inline void fbstring_core::reserveMedium(const size_t minCapacity) { } template -inline void fbstring_core::reserveSmall( +FOLLY_MALLOC_NOINLINE inline void fbstring_core::reserveSmall( size_t minCapacity, const bool disableSSO) { FBSTRING_ASSERT(category() == Category::isSmall); if (!disableSSO && minCapacity <= maxSmallSize) { @@ -1140,21 +1149,23 @@ public: assign(str, pos, n); } + FOLLY_MALLOC_NOINLINE /* implicit */ basic_fbstring(const value_type* s, const A& /*a*/ = A()) - : store_(s, basic_fbstring::traitsLength(s)) { - } + : store_(s, traitsLength(s)) {} + FOLLY_MALLOC_NOINLINE basic_fbstring(const value_type* s, size_type n, const A& /*a*/ = A()) : store_(s, n) { } + FOLLY_MALLOC_NOINLINE basic_fbstring(size_type n, value_type c, const A& /*a*/ = A()) { auto const pData = store_.expandNoinit(n); fbstring_detail::podFill(pData, pData + n, c); } template - basic_fbstring( + FOLLY_MALLOC_NOINLINE basic_fbstring( InIt begin, InIt end, typename std::enable_if< @@ -1164,6 +1175,7 @@ public: } // Specialization for const char*, const char* + FOLLY_MALLOC_NOINLINE basic_fbstring(const value_type* b, const value_type* e, const A& /*a*/ = A()) : store_(b, e - b) { } @@ -1175,12 +1187,12 @@ public: } // Construction from initialization list + FOLLY_MALLOC_NOINLINE basic_fbstring(std::initializer_list il) { assign(il.begin(), il.end()); } - ~basic_fbstring() noexcept { - } + ~basic_fbstring() noexcept {} basic_fbstring& operator=(const basic_fbstring& lhs); @@ -1352,7 +1364,7 @@ public: basic_fbstring& append(const value_type* s, size_type n); basic_fbstring& append(const value_type* s) { - return append(s, traits_type::length(s)); + return append(s, traitsLength(s)); } basic_fbstring& append(size_type n, value_type c); @@ -1386,7 +1398,7 @@ public: basic_fbstring& assign(const value_type* s, const size_type n); basic_fbstring& assign(const value_type* s) { - return assign(s, traits_type::length(s)); + return assign(s, traitsLength(s)); } basic_fbstring& assign(std::initializer_list il) { @@ -1416,7 +1428,7 @@ public: } basic_fbstring& insert(size_type pos, const value_type* s) { - return insert(pos, s, traits_type::length(s)); + return insert(pos, s, traitsLength(s)); } basic_fbstring& insert(size_type pos, size_type n, value_type c) { @@ -1525,7 +1537,7 @@ public: // Replaces at most n1 chars of *this, starting with pos, with chars from s basic_fbstring& replace(size_type pos, size_type n1, const value_type* s) { - return replace(pos, n1, s, traits_type::length(s)); + return replace(pos, n1, s, traitsLength(s)); } // Replaces at most n1 chars of *this, starting with pos, with n2 @@ -1551,7 +1563,7 @@ public: } basic_fbstring& replace(iterator i1, iterator i2, const value_type* s) { - return replace(i1, i2, s, traits_type::length(s)); + return replace(i1, i2, s, traitsLength(s)); } private: @@ -1654,7 +1666,7 @@ private: const; size_type find(const value_type* s, size_type pos = 0) const { - return find(s, pos, traits_type::length(s)); + return find(s, pos, traitsLength(s)); } size_type find (value_type c, size_type pos = 0) const { @@ -1668,7 +1680,7 @@ private: size_type rfind(const value_type* s, size_type pos, size_type n) const; size_type rfind(const value_type* s, size_type pos = npos) const { - return rfind(s, pos, traits_type::length(s)); + return rfind(s, pos, traitsLength(s)); } size_type rfind(value_type c, size_type pos = npos) const { @@ -1683,7 +1695,7 @@ private: const; size_type find_first_of(const value_type* s, size_type pos = 0) const { - return find_first_of(s, pos, traits_type::length(s)); + return find_first_of(s, pos, traitsLength(s)); } size_type find_first_of(value_type c, size_type pos = 0) const { @@ -1699,7 +1711,7 @@ private: size_type find_last_of (const value_type* s, size_type pos = npos) const { - return find_last_of(s, pos, traits_type::length(s)); + return find_last_of(s, pos, traitsLength(s)); } size_type find_last_of (value_type c, size_type pos = npos) const { @@ -1716,7 +1728,7 @@ private: size_type find_first_not_of(const value_type* s, size_type pos = 0) const { - return find_first_not_of(s, pos, traits_type::length(s)); + return find_first_not_of(s, pos, traitsLength(s)); } size_type find_first_not_of(value_type c, size_type pos = 0) const { @@ -1733,7 +1745,7 @@ private: size_type find_last_not_of(const value_type* s, size_type pos = npos) const { - return find_last_not_of(s, pos, traits_type::length(s)); + return find_last_not_of(s, pos, traitsLength(s)); } size_type find_last_not_of (value_type c, size_type pos = npos) const { @@ -1766,7 +1778,7 @@ private: int compare(size_type pos1, size_type n1, const value_type* s) const { - return compare(pos1, n1, s, traits_type::length(s)); + return compare(pos1, n1, s, traitsLength(s)); } int compare(size_type pos1, size_type n1, @@ -1788,9 +1800,9 @@ private: // Code from Jean-Francois Bastien (03/26/2007) int compare(const value_type* s) const { - // Could forward to compare(0, size(), s, traits_type::length(s)) + // Could forward to compare(0, size(), s, traitsLength(s)) // but that does two extra checks - const size_type n1(size()), n2(traits_type::length(s)); + const size_type n1(size()), n2(traitsLength(s)); const int r = traits_type::compare(data(), s, std::min(n1, n2)); return r != 0 ? r : n1 > n2 ? 1 : n1 < n2 ? -1 : 0; } @@ -1801,7 +1813,7 @@ private: }; template -inline typename basic_fbstring::size_type +FOLLY_MALLOC_NOINLINE inline typename basic_fbstring::size_type basic_fbstring::traitsLength(const value_type* s) { return s ? traits_type::length(s) : (std::__throw_logic_error( @@ -1891,8 +1903,8 @@ inline basic_fbstring& basic_fbstring::append( } template -inline basic_fbstring& basic_fbstring::append( - const value_type* s, size_type n) { +FOLLY_MALLOC_NOINLINE inline basic_fbstring& +basic_fbstring::append(const value_type* s, size_type n) { Invariant checker(*this); if (FBSTRING_UNLIKELY(!n)) { @@ -1942,11 +1954,10 @@ inline basic_fbstring& basic_fbstring::assign( } template -inline basic_fbstring& basic_fbstring::assign( - const value_type* s, const size_type n) { +FOLLY_MALLOC_NOINLINE inline basic_fbstring& +basic_fbstring::assign(const value_type* s, const size_type n) { Invariant checker(*this); - // s can alias this, we need to use podMove. if (n == 0) { resize(0); } else if (size() >= n) { diff --git a/folly/Malloc.h b/folly/Malloc.h index 8092ab0b..de75e451 100644 --- a/folly/Malloc.h +++ b/folly/Malloc.h @@ -111,6 +111,7 @@ __attribute__((__weak__)); #include #include +#include #include #ifdef _LIBSTDCXX_FBSTRING @@ -121,17 +122,21 @@ namespace folly { #endif // Cannot depend on Portability.h when _LIBSTDCXX_FBSTRING. -// Disabled for nvcc because it fails on attributes on lambdas. -#if defined(__GNUC__) && !defined(__NVCC__) +#if defined(__GNUC__) #define FOLLY_MALLOC_NOINLINE __attribute__((__noinline__)) +// This is for checked malloc-like functions (returns non-null pointer +// which cannot alias any outstanding pointer). +#define FOLLY_MALLOC_CHECKED_MALLOC \ + __attribute__((__returns_nonnull__, __malloc__)) #else #define FOLLY_MALLOC_NOINLINE +#define FOLLY_MALLOC_CHECKED_MALLOC #endif /** * Determine if we are using jemalloc or not. */ -inline bool usingJEMalloc() noexcept { +FOLLY_MALLOC_NOINLINE inline bool usingJEMalloc() noexcept { // Checking for rallocx != NULL is not sufficient; we may be in a dlopen()ed // module that depends on libjemalloc, so rallocx is resolved, but the main // program might be using a different memory allocator. @@ -140,7 +145,7 @@ inline bool usingJEMalloc() noexcept { // per-thread counter of allocated memory increases. This makes me // feel dirty inside. Also note that this requires jemalloc to have // been compiled with --enable-stats. - static const bool result = [] () FOLLY_MALLOC_NOINLINE noexcept { + static const bool result = [] () noexcept { // Some platforms (*cough* OSX *cough*) require weak symbol checks to be // in the form if (mallctl != nullptr). Not if (mallctl) or if (!mallctl) // (!!). http://goo.gl/xpmctm @@ -235,10 +240,11 @@ inline void* checkedRealloc(void* ptr, size_t size) { * routine just tries to call realloc() (thus benefitting of potential * copy-free coalescing) unless there's too much slack memory. */ -inline void* smartRealloc(void* p, - const size_t currentSize, - const size_t currentCapacity, - const size_t newCapacity) { +FOLLY_MALLOC_CHECKED_MALLOC FOLLY_MALLOC_NOINLINE inline void* smartRealloc( + void* p, + const size_t currentSize, + const size_t currentCapacity, + const size_t newCapacity) { assert(p); assert(currentSize <= currentCapacity && currentCapacity < newCapacity); -- 2.34.1