X-Git-Url: http://plrg.eecs.uci.edu/git/?p=folly.git;a=blobdiff_plain;f=folly%2FFBVector.h;h=428eba628d10b1629887c3fb09b94c181d1c3ff9;hp=e1319587727099bcad85ec531249d57a8599da0f;hb=eb7bc45f22e034751a76bf02445c669471e60780;hpb=ec0da6900c92e487781763dd1ca75a72e47dca7f diff --git a/folly/FBVector.h b/folly/FBVector.h index e1319587..428eba62 100644 --- a/folly/FBVector.h +++ b/folly/FBVector.h @@ -1,5 +1,5 @@ /* - * Copyright 2013 Facebook, Inc. + * Copyright 2017 Facebook, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,8 +22,7 @@ * optimizations for use with relocatable types and jemalloc. */ -#ifndef FOLLY_FBVECTOR_H -#define FOLLY_FBVECTOR_H +#pragma once //============================================================================= // headers @@ -36,119 +35,19 @@ #include #include -#include "folly/Likely.h" -#include "folly/Malloc.h" -#include "folly/Traits.h" - -#include - -// some files expected these from FBVector -#include -#include "folly/Foreach.h" -#include -#include +#include +#include +#include +#include +#include //============================================================================= // forward declaration -#ifdef FOLLY_BENCHMARK_USE_NS_IFOLLY -namespace Ifolly { -#else namespace folly { -#endif - template > - class fbvector; -} - -//============================================================================= -// compatibility - -#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ < 7 -// PLEASE UPGRADE TO GCC 4.7 or above -#define FOLLY_FBV_COMPATIBILITY_MODE -#endif - -#ifndef FOLLY_FBV_COMPATIBILITY_MODE - -namespace folly { - -template -struct fbv_allocator_traits - : std::allocator_traits {}; - -template -struct fbv_is_nothrow_move_constructible - : std::is_nothrow_move_constructible {}; - -template -struct fbv_is_nothrow_constructible - : std::is_nothrow_constructible {}; - -template -struct fbv_is_copy_constructible - : std::is_copy_constructible {}; - -} - -#else - -namespace folly { - -template -struct fbv_allocator_traits { - static_assert(sizeof(A) == 0, - "If you want to use a custom allocator, then you must upgrade to gcc 4.7"); - // for some old code that deals with this case, see D566719, diff number 10. -}; - -template -struct fbv_allocator_traits> { - typedef std::allocator A; - - typedef T* pointer; - typedef const T* const_pointer; - typedef size_t size_type; - - typedef std::false_type propagate_on_container_copy_assignment; - typedef std::false_type propagate_on_container_move_assignment; - typedef std::false_type propagate_on_container_swap; - - static pointer allocate(A& a, size_type n) { - return static_cast(::operator new(n * sizeof(T))); - } - static void deallocate(A& a, pointer p, size_type n) { - ::operator delete(p); - } - - template - static void construct(A& a, R* p, Args&&... args) { - new (p) R(std::forward(args)...); - } - template - static void destroy(A& a, R* p) { - p->~R(); - } - - static A select_on_container_copy_construction(const A& a) { - return a; - } -}; - -template -struct fbv_is_nothrow_move_constructible - : std::false_type {}; - -template -struct fbv_is_nothrow_constructible - : std::false_type {}; - -template -struct fbv_is_copy_constructible - : std::true_type {}; - -} - -#endif +template > +class fbvector; +} // namespace folly //============================================================================= // unrolling @@ -170,21 +69,16 @@ struct fbv_is_copy_constructible // // /////////////////////////////////////////////////////////////////////////////// -#ifdef FOLLY_BENCHMARK_USE_NS_IFOLLY -namespace Ifolly { -#else namespace folly { -#endif template -class fbvector : private boost::totally_ordered> { +class fbvector { //=========================================================================== //--------------------------------------------------------------------------- // implementation -private: - - typedef folly::fbv_allocator_traits A; + private: + typedef std::allocator_traits A; struct Impl : public Allocator { // typedefs @@ -196,16 +90,16 @@ private: // constructors Impl() : Allocator(), b_(nullptr), e_(nullptr), z_(nullptr) {} - Impl(const Allocator& a) + /* implicit */ Impl(const Allocator& a) : Allocator(a), b_(nullptr), e_(nullptr), z_(nullptr) {} - Impl(Allocator&& a) + /* implicit */ Impl(Allocator&& a) : Allocator(std::move(a)), b_(nullptr), e_(nullptr), z_(nullptr) {} - Impl(size_type n, const Allocator& a = Allocator()) + /* implicit */ Impl(size_type n, const Allocator& a = Allocator()) : Allocator(a) { init(n); } - Impl(Impl&& other) + Impl(Impl&& other) noexcept : Allocator(std::move(other)), b_(other.b_), e_(other.e_), z_(other.z_) { other.b_ = other.e_ = other.z_ = nullptr; } @@ -221,7 +115,7 @@ private: if (usingStdAllocator::value) { return static_cast(malloc(n * sizeof(T))); } else { - return folly::fbv_allocator_traits::allocate(*this, n); + return std::allocator_traits::allocate(*this, n); } } @@ -229,7 +123,7 @@ private: if (usingStdAllocator::value) { free(p); } else { - folly::fbv_allocator_traits::deallocate(*this, p, n); + std::allocator_traits::deallocate(*this, p, n); } } @@ -252,7 +146,7 @@ private: S_destroy_range_a(*this, b_, e_); } - D_deallocate(b_, z_ - b_); + D_deallocate(b_, size_type(z_ - b_)); } } @@ -267,8 +161,7 @@ private: } } - void - set(pointer newB, size_type newSize, size_type newCap) { + void set(pointer newB, size_type newSize, size_type newCap) { z_ = newB + newCap; e_ = newB + newSize; b_ = newB; @@ -291,15 +184,16 @@ private: static void swap(Impl& a, Impl& b) { using std::swap; - if (!usingStdAllocator::value) swap(a, b); + if (!usingStdAllocator::value) { + swap(a, b); + } a.swapData(b); } //=========================================================================== //--------------------------------------------------------------------------- // types and constants -public: - + public: typedef T value_type; typedef value_type& reference; typedef const value_type& const_reference; @@ -313,10 +207,9 @@ public: typedef std::reverse_iterator reverse_iterator; typedef std::reverse_iterator const_reverse_iterator; -private: - + private: typedef std::integral_constant::value && + IsTriviallyCopyable::value && sizeof(T) <= 16 // don't force large structures to be passed by value > should_pass_by_value; typedef typename std::conditional< @@ -333,8 +226,7 @@ private: //=========================================================================== //--------------------------------------------------------------------------- // allocator helpers -private: - + private: //--------------------------------------------------------------------------- // allocate @@ -360,7 +252,7 @@ private: if (usingStdAllocator::value) { new (p) U(std::forward(args)...); } else { - folly::fbv_allocator_traits::construct( + std::allocator_traits::construct( impl_, p, std::forward(args)...); } } @@ -372,7 +264,7 @@ private: template static void S_construct_a(Allocator& a, U* p, Args&&... args) { - folly::fbv_allocator_traits::construct( + std::allocator_traits::construct( a, p, std::forward(args)...); } @@ -384,7 +276,7 @@ private: if (usingStdAllocator::value) { *p = arg; } else { - folly::fbv_allocator_traits::construct(impl_, p, arg); + std::allocator_traits::construct(impl_, p, arg); } } @@ -397,7 +289,7 @@ private: template ::value>::type> static void S_construct_a(Allocator& a, U* p, U arg) { - folly::fbv_allocator_traits::construct(a, p, arg); + std::allocator_traits::construct(a, p, arg); } // const& optimization @@ -407,7 +299,7 @@ private: if (usingStdAllocator::value) { new (p) U(value); } else { - folly::fbv_allocator_traits::construct(impl_, p, value); + std::allocator_traits::construct(impl_, p, value); } } @@ -420,7 +312,7 @@ private: template ::value>::type> static void S_construct_a(Allocator& a, U* p, const U& value) { - folly::fbv_allocator_traits::construct(a, p, value); + std::allocator_traits::construct(a, p, value); } //--------------------------------------------------------------------------- @@ -428,17 +320,18 @@ private: void M_destroy(T* p) noexcept { if (usingStdAllocator::value) { - if (!boost::has_trivial_destructor::value) p->~T(); + if (!std::is_trivially_destructible::value) { + p->~T(); + } } else { - folly::fbv_allocator_traits::destroy(impl_, p); + std::allocator_traits::destroy(impl_, p); } } //=========================================================================== //--------------------------------------------------------------------------- // algorithmic helpers -private: - + private: //--------------------------------------------------------------------------- // destroy_range @@ -460,13 +353,14 @@ private: // allocator static void S_destroy_range_a(Allocator& a, T* first, T* last) noexcept { - for (; first != last; ++first) - folly::fbv_allocator_traits::destroy(a, first); + for (; first != last; ++first) { + std::allocator_traits::destroy(a, first); + } } // optimized static void S_destroy_range(T* first, T* last) noexcept { - if (!boost::has_trivial_destructor::value) { + if (!std::is_trivially_destructible::value) { // EXPERIMENTAL DATA on fbvector> (where each vector has // size 0). // The unrolled version seems to work faster for small to medium sized @@ -522,9 +416,10 @@ private: auto b = dest; auto e = dest + sz; try { - for (; b != e; ++b) - folly::fbv_allocator_traits::construct(a, b, + for (; b != e; ++b) { + std::allocator_traits::construct(a, b, std::forward(args)...); + } } catch (...) { S_destroy_range_a(a, dest, b); throw; @@ -534,15 +429,21 @@ private: // optimized static void S_uninitialized_fill_n(T* dest, size_type n) { if (folly::IsZeroInitializable::value) { - std::memset(dest, 0, sizeof(T) * n); + if (LIKELY(n != 0)) { + std::memset(dest, 0, sizeof(T) * n); + } } else { auto b = dest; auto e = dest + n; try { - for (; b != e; ++b) S_construct(b); + for (; b != e; ++b) { + S_construct(b); + } } catch (...) { --b; - for (; b >= dest; --b) b->~T(); + for (; b >= dest; --b) { + b->~T(); + } throw; } } @@ -552,7 +453,9 @@ private: auto b = dest; auto e = dest + n; try { - for (; b != e; ++b) S_construct(b, value); + for (; b != e; ++b) { + S_construct(b, value); + } } catch (...) { S_destroy_range(dest, b); throw; @@ -604,8 +507,9 @@ private: S_uninitialized_copy_a(Allocator& a, T* dest, It first, It last) { auto b = dest; try { - for (; first != last; ++first, ++b) - folly::fbv_allocator_traits::construct(a, b, *first); + for (; first != last; ++first, ++b) { + std::allocator_traits::construct(a, b, *first); + } } catch (...) { S_destroy_range_a(a, dest, b); throw; @@ -617,8 +521,9 @@ private: static void S_uninitialized_copy(T* dest, It first, It last) { auto b = dest; try { - for (; first != last; ++first, ++b) + for (; first != last; ++first, ++b) { S_construct(b, *first); + } } catch (...) { S_destroy_range(dest, b); throw; @@ -627,7 +532,9 @@ private: static void S_uninitialized_copy_bits(T* dest, const T* first, const T* last) { - std::memcpy(dest, first, (last - first) * sizeof(T)); + if (last != first) { + std::memcpy((void*)dest, (void*)first, (last - first) * sizeof(T)); + } } static void @@ -635,7 +542,9 @@ private: std::move_iterator last) { T* bFirst = first.base(); T* bLast = last.base(); - std::memcpy(dest, bFirst, (bLast - bFirst) * sizeof(T)); + if (bLast != bFirst) { + std::memcpy((void*)dest, (void*)bFirst, (bLast - bFirst) * sizeof(T)); + } } template @@ -654,13 +563,15 @@ private: template static It S_copy_n(T* dest, It first, size_type n) { auto e = dest + n; - for (; dest != e; ++dest, ++first) *dest = *first; + for (; dest != e; ++dest, ++first) { + *dest = *first; + } return first; } static const T* S_copy_n(T* dest, const T* first, size_type n) { if (folly::IsTriviallyCopyable::value) { - std::memcpy(dest, first, n * sizeof(T)); + std::memcpy((void*)dest, (void*)first, n * sizeof(T)); return first + n; } else { return S_copy_n(dest, first, n); @@ -671,7 +582,7 @@ private: S_copy_n(T* dest, std::move_iterator mIt, size_type n) { if (folly::IsTriviallyCopyable::value) { T* first = mIt.base(); - std::memcpy(dest, first, n * sizeof(T)); + std::memcpy((void*)dest, (void*)first, n * sizeof(T)); return std::make_move_iterator(first + n); } else { return S_copy_n>(dest, mIt, n); @@ -681,8 +592,7 @@ private: //=========================================================================== //--------------------------------------------------------------------------- // relocation helpers -private: - + private: // Relocation is divided into three parts: // // 1: relocate_move @@ -730,9 +640,9 @@ private: > relocate_use_memcpy; typedef std::integral_constant::value + (std::is_nothrow_move_constructible::value && usingStdAllocator::value) - || !folly::fbv_is_copy_constructible::value + || !std::is_copy_constructible::value > relocate_use_move; // move @@ -741,7 +651,9 @@ private: } void relocate_move_or_memcpy(T* dest, T* first, T* last, std::true_type) { - std::memcpy(dest, first, (last - first) * sizeof(T)); + if (first != nullptr) { + std::memcpy((void*)dest, (void*)first, (last - first) * sizeof(T)); + } } void relocate_move_or_memcpy(T* dest, T* first, T* last, std::false_type) { @@ -757,7 +669,7 @@ private: } // done - void relocate_done(T* dest, T* first, T* last) noexcept { + void relocate_done(T* /*dest*/, T* first, T* last) noexcept { if (folly::IsRelocatable::value && usingStdAllocator::value) { // used memcpy; data has been relocated, do not call destructor } else { @@ -769,11 +681,11 @@ private: void relocate_undo(T* dest, T* first, T* last) noexcept { if (folly::IsRelocatable::value && usingStdAllocator::value) { // used memcpy, old data is still valid, nothing to do - } else if (folly::fbv_is_nothrow_move_constructible::value && + } else if (std::is_nothrow_move_constructible::value && usingStdAllocator::value) { // noexcept move everything back, aka relocate_move relocate_move(first, dest, dest + (last - first)); - } else if (!folly::fbv_is_copy_constructible::value) { + } else if (!std::is_copy_constructible::value) { // weak guarantee D_destroy_range_a(dest, dest + (last - first)); } else { @@ -786,8 +698,7 @@ private: //=========================================================================== //--------------------------------------------------------------------------- // construct/copy/destroy -public: - + public: fbvector() = default; explicit fbvector(const Allocator& a) : impl_(a) {} @@ -803,12 +714,7 @@ public: template ::iterator_category> fbvector(It first, It last, const Allocator& a = Allocator()) - #ifndef FOLLY_FBV_COMPATIBILITY_MODE : fbvector(first, last, a, Category()) {} - #else - : impl_(std::distance(first, last), a) - { fbvector_init(first, last, Category()); } - #endif fbvector(const fbvector& other) : impl_(other.size(), A::select_on_container_copy_construction(other.impl_)) @@ -817,14 +723,9 @@ public: fbvector(fbvector&& other) noexcept : impl_(std::move(other.impl_)) {} fbvector(const fbvector& other, const Allocator& a) - #ifndef FOLLY_FBV_COMPATIBILITY_MODE : fbvector(other.begin(), other.end(), a) {} - #else - : impl_(other.size(), a) - { fbvector_init(other.begin(), other.end(), std::forward_iterator_tag()); } - #endif - fbvector(fbvector&& other, const Allocator& a) : impl_(a) { + /* may throw */ fbvector(fbvector&& other, const Allocator& a) : impl_(a) { if (impl_ == other.impl_) { impl_.swapData(other.impl_); } else { @@ -834,17 +735,14 @@ public: } fbvector(std::initializer_list il, const Allocator& a = Allocator()) - #ifndef FOLLY_FBV_COMPATIBILITY_MODE : fbvector(il.begin(), il.end(), a) {} - #else - : impl_(std::distance(il.begin(), il.end()), a) - { fbvector_init(il.begin(), il.end(), std::forward_iterator_tag()); } - #endif ~fbvector() = default; // the cleanup occurs in impl_ fbvector& operator=(const fbvector& other) { - if (UNLIKELY(this == &other)) return *this; + if (UNLIKELY(this == &other)) { + return *this; + } if (!usingStdAllocator::value && A::propagate_on_container_copy_assignment::value) { @@ -860,7 +758,9 @@ public: } fbvector& operator=(fbvector&& other) { - if (UNLIKELY(this == &other)) return *this; + if (UNLIKELY(this == &other)) { + return *this; + } moveFrom(std::move(other), moveIsSwap()); return *this; } @@ -906,36 +806,25 @@ public: return impl_; } -private: - - #ifndef FOLLY_FBV_COMPATIBILITY_MODE + private: // contract dispatch for iterator types fbvector(It first, It last) template fbvector(ForwardIterator first, ForwardIterator last, const Allocator& a, std::forward_iterator_tag) - : impl_(std::distance(first, last), a) - { M_uninitialized_copy_e(first, last); } - - template - fbvector(InputIterator first, InputIterator last, - const Allocator& a, std::input_iterator_tag) - : impl_(a) - { for (; first != last; ++first) emplace_back(*first); } - - #else - // contract dispatch for iterator types without constructor forwarding - template - void - fbvector_init(ForwardIterator first, ForwardIterator last, - std::forward_iterator_tag) + : impl_(size_type(std::distance(first, last)), a) { M_uninitialized_copy_e(first, last); } template - void - fbvector_init(InputIterator first, InputIterator last, - std::input_iterator_tag) - { for (; first != last; ++first) emplace_back(*first); } - #endif + fbvector( + InputIterator first, + InputIterator last, + const Allocator& a, + std::input_iterator_tag) + : impl_(a) { + for (; first != last; ++first) { + emplace_back(*first); + } + } // contract dispatch for allocator movement in operator=(fbvector&&) void @@ -955,7 +844,7 @@ private: template void assign(ForwardIterator first, ForwardIterator last, std::forward_iterator_tag) { - auto const newSize = std::distance(first, last); + const auto newSize = size_type(std::distance(first, last)); if (newSize > capacity()) { impl_.reset(newSize); M_uninitialized_copy_e(first, last); @@ -978,13 +867,17 @@ private: if (p != impl_.e_) { M_destroy_range_e(p); } else { - for (; first != last; ++first) emplace_back(*first); + for (; first != last; ++first) { + emplace_back(*first); + } } } // contract dispatch for aliasing under VT optimization bool dataIsInternalAndNotVT(const T& t) { - if (should_pass_by_value::value) return false; + if (should_pass_by_value::value) { + return false; + } return dataIsInternal(t); } bool dataIsInternal(const T& t) { @@ -996,8 +889,7 @@ private: //=========================================================================== //--------------------------------------------------------------------------- // iterators -public: - + public: iterator begin() noexcept { return impl_.b_; } @@ -1039,10 +931,9 @@ public: //=========================================================================== //--------------------------------------------------------------------------- // capacity -public: - + public: size_type size() const noexcept { - return impl_.e_ - impl_.b_; + return size_type(impl_.e_ - impl_.b_); } size_type max_size() const noexcept { @@ -1073,7 +964,7 @@ public: } size_type capacity() const noexcept { - return impl_.z_ - impl_.b_; + return size_type(impl_.z_ - impl_.b_); } bool empty() const noexcept { @@ -1081,8 +972,12 @@ public: } void reserve(size_type n) { - if (n <= capacity()) return; - if (impl_.b_ && reserve_in_place(n)) return; + if (n <= capacity()) { + return; + } + if (impl_.b_ && reserve_in_place(n)) { + return; + } auto newCap = folly::goodMallocSize(n * sizeof(T)) / sizeof(T); auto newB = M_allocate(newCap); @@ -1092,25 +987,34 @@ public: M_deallocate(newB, newCap); throw; } - if (impl_.b_) - M_deallocate(impl_.b_, impl_.z_ - impl_.b_); + if (impl_.b_) { + M_deallocate(impl_.b_, size_type(impl_.z_ - impl_.b_)); + } impl_.z_ = newB + newCap; impl_.e_ = newB + (impl_.e_ - impl_.b_); impl_.b_ = newB; } void shrink_to_fit() noexcept { + if (empty()) { + impl_.reset(); + return; + } + auto const newCapacityBytes = folly::goodMallocSize(size() * sizeof(T)); auto const newCap = newCapacityBytes / sizeof(T); auto const oldCap = capacity(); - if (newCap >= oldCap) return; + if (newCap >= oldCap) { + return; + } void* p = impl_.b_; - if ((rallocm && usingStdAllocator::value) && + // xallocx() will shrink to precisely newCapacityBytes (which was generated + // by goodMallocSize()) if it successfully shrinks in place. + if ((usingJEMalloc() && usingStdAllocator::value) && newCapacityBytes >= folly::jemallocMinInPlaceExpandable && - rallocm(&p, NULL, newCapacityBytes, 0, ALLOCM_NO_MOVE) - == ALLOCM_SUCCESS) { + xallocx(p, newCapacityBytes, 0, 0) == newCapacityBytes) { impl_.z_ += newCap - oldCap; } else { T* newB; // intentionally uninitialized @@ -1125,27 +1029,30 @@ public: } catch (...) { return; } - if (impl_.b_) - M_deallocate(impl_.b_, impl_.z_ - impl_.b_); + if (impl_.b_) { + M_deallocate(impl_.b_, size_type(impl_.z_ - impl_.b_)); + } impl_.z_ = newB + newCap; impl_.e_ = newB + (impl_.e_ - impl_.b_); impl_.b_ = newB; } } -private: - + private: bool reserve_in_place(size_type n) { - if (!usingStdAllocator::value || !rallocm) return false; + if (!usingStdAllocator::value || !usingJEMalloc()) { + return false; + } // jemalloc can never grow in place blocks smaller than 4096 bytes. if ((impl_.z_ - impl_.b_) * sizeof(T) < - folly::jemallocMinInPlaceExpandable) return false; + folly::jemallocMinInPlaceExpandable) { + return false; + } auto const newCapacityBytes = folly::goodMallocSize(n * sizeof(T)); void* p = impl_.b_; - if (rallocm(&p, NULL, newCapacityBytes, 0, ALLOCM_NO_MOVE) - == ALLOCM_SUCCESS) { + if (xallocx(p, newCapacityBytes, 0, 0) == newCapacityBytes) { impl_.z_ = impl_.b_ + newCapacityBytes / sizeof(T); return true; } @@ -1155,8 +1062,7 @@ private: //=========================================================================== //--------------------------------------------------------------------------- // element access -public: - + public: reference operator[](size_type n) { assert(n < size()); return impl_.b_[n]; @@ -1167,7 +1073,7 @@ public: } const_reference at(size_type n) const { if (UNLIKELY(n >= size())) { - throw std::out_of_range("fbvector: index is greater than size."); + std::__throw_out_of_range("fbvector: index is greater than size."); } return (*this)[n]; } @@ -1195,8 +1101,7 @@ public: //=========================================================================== //--------------------------------------------------------------------------- // data access -public: - + public: T* data() noexcept { return impl_.b_; } @@ -1207,8 +1112,7 @@ public: //=========================================================================== //--------------------------------------------------------------------------- // modifiers (common) -public: - + public: template void emplace_back(Args&&... args) { if (impl_.e_ != impl_.z_) { @@ -1246,27 +1150,27 @@ public: } void swap(fbvector& other) noexcept { - if (!usingStdAllocator::value && - A::propagate_on_container_swap::value) + if (!usingStdAllocator::value && A::propagate_on_container_swap::value) { swap(impl_, other.impl_); - else impl_.swapData(other.impl_); + } else { + impl_.swapData(other.impl_); + } } void clear() noexcept { M_destroy_range_e(impl_.b_); } -private: - + private: // std::vector implements a similar function with a different growth // strategy: empty() ? 1 : capacity() * 2. // // fbvector grows differently on two counts: // // (1) initial size - // Instead of grwoing to size 1 from empty, and fbvector allocates at - // least 64 bytes. You may still use reserve to reserve a lesser amount - // of memory. + // Instead of growing to size 1 from empty, fbvector allocates at least + // 64 bytes. You may still use reserve to reserve a lesser amount of + // memory. // (2) 1.5x // For medium-sized vectors, the growth strategy is 1.5x. See the docs // for details. @@ -1277,14 +1181,16 @@ private: // size_type computePushBackCapacity() const { - return empty() ? std::max(64 / sizeof(T), size_type(1)) - : capacity() < folly::jemallocMinInPlaceExpandable / sizeof(T) - ? capacity() * 2 - : sizeof(T) > folly::jemallocMinInPlaceExpandable / 2 && capacity() == 1 - ? 2 - : capacity() > 4096 * 32 / sizeof(T) - ? capacity() * 2 - : (capacity() * 3 + 1) / 2; + if (capacity() == 0) { + return std::max(64 / sizeof(T), size_type(1)); + } + if (capacity() < folly::jemallocMinInPlaceExpandable / sizeof(T)) { + return capacity() * 2; + } + if (capacity() > 4096 * 32 / sizeof(T)) { + return capacity() * 2; + } + return (capacity() * 3 + 1) / 2; } template @@ -1293,8 +1199,7 @@ private: //=========================================================================== //--------------------------------------------------------------------------- // modifiers (erase) -public: - + public: iterator erase(const_iterator position) { return erase(position, position + 1); } @@ -1309,7 +1214,7 @@ public: if (folly::IsRelocatable::value && usingStdAllocator::value) { D_destroy_range_a((iterator)first, (iterator)last); if (last - first >= cend() - last) { - std::memcpy((iterator)first, last, (cend() - last) * sizeof(T)); + std::memcpy((void*)first, (void*)last, (cend() - last) * sizeof(T)); } else { std::memmove((iterator)first, last, (cend() - last) * sizeof(T)); } @@ -1328,8 +1233,7 @@ public: //=========================================================================== //--------------------------------------------------------------------------- // modifiers (insert) -private: // we have the private section first because it defines some macros - + private: // we have the private section first because it defines some macros bool isValid(const_iterator it) { return cbegin() <= it && it <= cend(); } @@ -1387,7 +1291,7 @@ private: // we have the private section first because it defines some macros // These three functions, make_window, wrap_frame, and // insert_use_fresh_memory, can be combined into a uniform interface. // Since that interface involves a lot of case-work, it is built into - // some macros: FOLLY_FBVECTOR_INSERT_(START|TRY|END) + // some macros: FOLLY_FBVECTOR_INSERT_(PRE|START|TRY|END) // Macros are used in an attempt to let GCC perform better optimizations, // especially control flow optimization. // @@ -1396,11 +1300,8 @@ private: // we have the private section first because it defines some macros // window void make_window(iterator position, size_type n) { - assert(isValid(position)); - assert(size() + n <= capacity()); - assert(n != 0); - - auto tail = std::distance(position, impl_.e_); + // The result is guaranteed to be non-negative, so use an unsigned type: + size_type tail = size_type(std::distance(position, impl_.e_)); if (tail <= n) { relocate_move(position + n, position, impl_.e_); @@ -1412,9 +1313,15 @@ private: // we have the private section first because it defines some macros impl_.e_ += n; } else { D_uninitialized_move_a(impl_.e_, impl_.e_ - n, impl_.e_); + try { + std::copy_backward(std::make_move_iterator(position), + std::make_move_iterator(impl_.e_ - n), impl_.e_); + } catch (...) { + D_destroy_range_a(impl_.e_ - n, impl_.e_ + n); + impl_.e_ -= n; + throw; + } impl_.e_ += n; - std::copy_backward(std::make_move_iterator(position), - std::make_move_iterator(impl_.e_ - n), impl_.e_); D_destroy_range_a(position, position + n); } } @@ -1446,14 +1353,20 @@ private: // we have the private section first because it defines some macros //--------------------------------------------------------------------------- // use fresh? - bool insert_use_fresh(const_iterator cposition, size_type n) { - if (cposition == cend()) { - if (size() + n <= capacity()) return false; - if (reserve_in_place(size() + n)) return false; + bool insert_use_fresh(bool at_end, size_type n) { + if (at_end) { + if (size() + n <= capacity()) { + return false; + } + if (reserve_in_place(size() + n)) { + return false; + } return true; } - if (size() + n > capacity()) return true; + if (size() + n > capacity()) { + return true; + } return false; } @@ -1461,99 +1374,125 @@ private: // we have the private section first because it defines some macros //--------------------------------------------------------------------------- // interface - #define FOLLY_FBVECTOR_INSERT_START(cpos, n) \ - assert(isValid(cpos)); \ - T* position = const_cast(cpos); \ - size_type idx = std::distance(impl_.b_, position); \ - bool fresh = insert_use_fresh(position, n); \ - T* b; \ - size_type newCap = 0; \ - \ - if (fresh) { \ - newCap = computeInsertCapacity(n); \ - b = M_allocate(newCap); \ - } else { \ - make_window(position, n); \ - b = impl_.b_; \ - } \ - \ - T* start = b + idx; \ - \ - try { \ - - // construct the inserted elements - - #define FOLLY_FBVECTOR_INSERT_TRY(cpos, n) \ - } catch (...) { \ - if (fresh) { \ - M_deallocate(b, newCap); \ - } else { \ - undo_window(position, n); \ - } \ - throw; \ - } \ - \ - if (fresh) { \ - try { \ - wrap_frame(b, idx, n); \ - } catch (...) { \ - - - // delete the inserted elements (exception has been thrown) - - #define FOLLY_FBVECTOR_INSERT_END(cpos, n) \ - M_deallocate(b, newCap); \ - throw; \ - } \ - if (impl_.b_) M_deallocate(impl_.b_, capacity()); \ - impl_.set(b, size() + n, newCap); \ - return impl_.b_ + idx; \ - } else { \ - return position; \ - } \ + template < + typename IsInternalFunc, + typename InsertInternalFunc, + typename ConstructFunc, + typename DestroyFunc> + iterator do_real_insert( + const_iterator cpos, + size_type n, + IsInternalFunc&& isInternalFunc, + InsertInternalFunc&& insertInternalFunc, + ConstructFunc&& constructFunc, + DestroyFunc&& destroyFunc) { + if (n == 0) { + return iterator(cpos); + } + bool at_end = cpos == cend(); + bool fresh = insert_use_fresh(at_end, n); + if (!at_end) { + if (!fresh && isInternalFunc()) { + // check for internal data (technically not required by the standard) + return insertInternalFunc(); + } + assert(isValid(cpos)); + } + T* position = const_cast(cpos); + size_type idx = size_type(std::distance(impl_.b_, position)); + T* b; + size_type newCap; /* intentionally uninitialized */ - //--------------------------------------------------------------------------- - // insert functions -public: + if (fresh) { + newCap = computeInsertCapacity(n); + b = M_allocate(newCap); + } else { + if (!at_end) { + make_window(position, n); + } else { + impl_.e_ += n; + } + b = impl_.b_; + } + T* start = b + idx; + try { + // construct the inserted elements + constructFunc(start); + } catch (...) { + if (fresh) { + M_deallocate(b, newCap); + } else { + if (!at_end) { + undo_window(position, n); + } else { + impl_.e_ -= n; + } + } + throw; + } + + if (fresh) { + try { + wrap_frame(b, idx, n); + } catch (...) { + // delete the inserted elements (exception has been thrown) + destroyFunc(start); + M_deallocate(b, newCap); + throw; + } + if (impl_.b_) { + M_deallocate(impl_.b_, capacity()); + } + impl_.set(b, size() + n, newCap); + return impl_.b_ + idx; + } else { + return position; + } + } + + public: template iterator emplace(const_iterator cpos, Args&&... args) { - FOLLY_FBVECTOR_INSERT_START(cpos, 1) - M_construct(start, std::forward(args)...); - FOLLY_FBVECTOR_INSERT_TRY(cpos, 1) - M_destroy(start); - FOLLY_FBVECTOR_INSERT_END(cpos, 1) + return do_real_insert( + cpos, + 1, + [&] { return false; }, + [&] { return iterator{}; }, + [&](iterator start) { + M_construct(start, std::forward(args)...); + }, + [&](iterator start) { M_destroy(start); }); } iterator insert(const_iterator cpos, const T& value) { - if (dataIsInternal(value)) return insert(cpos, T(value)); - - FOLLY_FBVECTOR_INSERT_START(cpos, 1) - M_construct(start, value); - FOLLY_FBVECTOR_INSERT_TRY(cpos, 1) - M_destroy(start); - FOLLY_FBVECTOR_INSERT_END(cpos, 1) + return do_real_insert( + cpos, + 1, + [&] { return dataIsInternal(value); }, + [&] { return insert(cpos, T(value)); }, + [&](iterator start) { M_construct(start, value); }, + [&](iterator start) { M_destroy(start); }); } iterator insert(const_iterator cpos, T&& value) { - if (dataIsInternal(value)) return insert(cpos, T(std::move(value))); - - FOLLY_FBVECTOR_INSERT_START(cpos, 1) - M_construct(start, std::move(value)); - FOLLY_FBVECTOR_INSERT_TRY(cpos, 1) - M_destroy(start); - FOLLY_FBVECTOR_INSERT_END(cpos, 1) + return do_real_insert( + cpos, + 1, + [&] { return dataIsInternal(value); }, + [&] { return insert(cpos, T(std::move(value))); }, + [&](iterator start) { M_construct(start, std::move(value)); }, + [&](iterator start) { M_destroy(start); }); } iterator insert(const_iterator cpos, size_type n, VT value) { - if (n == 0) return (iterator)cpos; - if (dataIsInternalAndNotVT(value)) return insert(cpos, n, T(value)); - - FOLLY_FBVECTOR_INSERT_START(cpos, n) - D_uninitialized_fill_n_a(start, n, value); - FOLLY_FBVECTOR_INSERT_TRY(cpos, n) - D_destroy_range_a(start, start + n); - FOLLY_FBVECTOR_INSERT_END(cpos, n) + return do_real_insert( + cpos, + n, + [&] { return dataIsInternalAndNotVT(value); }, + [&] { return insert(cpos, n, T(value)); }, + [&](iterator start) { D_uninitialized_fill_n_a(start, n, value); }, + [&](iterator start) { D_destroy_range_a(start, start + n); }); } template iterator insert(const_iterator cpos, FIt first, FIt last, std::forward_iterator_tag) { - size_type n = std::distance(first, last); - if (n == 0) return (iterator)cpos; - - FOLLY_FBVECTOR_INSERT_START(cpos, n) - D_uninitialized_copy_a(start, first, last); - FOLLY_FBVECTOR_INSERT_TRY(cpos, n) - D_destroy_range_a(start, start + n); - FOLLY_FBVECTOR_INSERT_END(cpos, n) + size_type n = size_type(std::distance(first, last)); + return do_real_insert( + cpos, + n, + [&] { return false; }, + [&] { return iterator{}; }, + [&](iterator start) { D_uninitialized_copy_a(start, first, last); }, + [&](iterator start) { D_destroy_range_a(start, start + n); }); } template @@ -1594,7 +1532,9 @@ private: std::make_move_iterator(end()), A::select_on_container_copy_construction(impl_)); M_destroy_range_e(position); - for (; first != last; ++first) emplace_back(*first); + for (; first != last; ++first) { + emplace_back(*first); + } insert(cend(), std::make_move_iterator(storage.begin()), std::make_move_iterator(storage.end())); return impl_.b_ + idx; @@ -1602,23 +1542,37 @@ private: //=========================================================================== //--------------------------------------------------------------------------- - // lexicographical functions (others from boost::totally_ordered superclass) -public: - + // lexicographical functions + public: bool operator==(const fbvector& other) const { return size() == other.size() && std::equal(begin(), end(), other.begin()); } + bool operator!=(const fbvector& other) const { + return !(*this == other); + } + bool operator<(const fbvector& other) const { return std::lexicographical_compare( begin(), end(), other.begin(), other.end()); } + bool operator>(const fbvector& other) const { + return other < *this; + } + + bool operator<=(const fbvector& other) const { + return !(*this > other); + } + + bool operator>=(const fbvector& other) const { + return !(*this < other); + } + //=========================================================================== //--------------------------------------------------------------------------- // friends -private: - + private: template friend _T* relinquish(fbvector<_T, _A>&); @@ -1638,17 +1592,17 @@ void fbvector::emplace_back_aux(Args&&... args) { size_type byte_sz = folly::goodMallocSize( computePushBackCapacity() * sizeof(T)); if (usingStdAllocator::value - && rallocm + && usingJEMalloc() && ((impl_.z_ - impl_.b_) * sizeof(T) >= folly::jemallocMinInPlaceExpandable)) { // Try to reserve in place. - // Ask rallocm to allocate in place at least size()+1 and at most sz space. - // rallocm will allocate as much as possible within that range, which + // Ask xallocx to allocate in place at least size()+1 and at most sz space. + // xallocx will allocate as much as possible within that range, which // is the best possible outcome: if sz space is available, take it all, // otherwise take as much as possible. If nothing is available, then fail. // In this fashion, we never relocate if there is a possibility of - // expanding in place, and we never relocate by less than the desired - // amount unless we cannot expand further. Hence we will not relocate + // expanding in place, and we never reallocate by less than the desired + // amount unless we cannot expand further. Hence we will not reallocate // sub-optimally twice in a row (modulo the blocking memory being freed). size_type lower = folly::goodMallocSize(sizeof(T) + size() * sizeof(T)); size_type upper = byte_sz; @@ -1657,8 +1611,7 @@ void fbvector::emplace_back_aux(Args&&... args) { void* p = impl_.b_; size_t actual; - if (rallocm(&p, &actual, lower, extra, ALLOCM_NO_MOVE) - == ALLOCM_SUCCESS) { + if ((actual = xallocx(p, lower, extra, 0)) >= lower) { impl_.z_ = impl_.b_ + actual / sizeof(T); M_construct(impl_.e_, std::forward(args)...); ++impl_.e_; @@ -1694,7 +1647,9 @@ void fbvector::emplace_back_aux(Args&&... args) { M_deallocate(newB, sz); throw; } - if (impl_.b_) M_deallocate(impl_.b_, size()); + if (impl_.b_) { + M_deallocate(impl_.b_, size()); + } impl_.b_ = newB; impl_.e_ = newE; impl_.z_ = newB + sz; @@ -1713,6 +1668,16 @@ void swap(fbvector& lhs, fbvector& rhs) noexcept { //----------------------------------------------------------------------------- // other +namespace detail { + +// Format support. +template +struct IndexableTraits> + : public IndexableTraitsSeq> { +}; + +} // namespace detail + template void compactResize(fbvector* v, size_t sz) { v->resize(sz); @@ -1758,5 +1723,3 @@ void attach(fbvector& v, T* data, size_t sz, size_t cap) { } } // namespace folly - -#endif // FOLLY_FBVECTOR_H