X-Git-Url: http://plrg.eecs.uci.edu/git/?a=blobdiff_plain;f=folly%2Fexperimental%2Fhazptr%2Fhazptr-impl.h;h=83f43b6f005e68d8e2097fd89fb5004e5fb9ae7b;hb=028415ba5c1fee579d249a97dd49752fe3c7348b;hp=6b19bd8aec8d0a3adf9729bac33007e173c6bf83;hpb=0f5638fe81b9dafd9a222bc5e9b1047692e0326e;p=folly.git diff --git a/folly/experimental/hazptr/hazptr-impl.h b/folly/experimental/hazptr/hazptr-impl.h index 6b19bd8a..83f43b6f 100644 --- a/folly/experimental/hazptr/hazptr-impl.h +++ b/folly/experimental/hazptr/hazptr-impl.h @@ -1,5 +1,5 @@ /* - * Copyright 2017 Facebook, Inc. + * Copyright 2017-present Facebook, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - /* override-include-guard */ #ifndef HAZPTR_H #error "This should only be included by hazptr.h" @@ -41,6 +40,10 @@ #define HAZPTR_PRIV true #endif +#ifndef HAZPTR_PRIV_THRESHOLD +#define HAZPTR_PRIV_THRESHOLD 20 +#endif + #ifndef HAZPTR_ONE_DOMAIN #define HAZPTR_ONE_DOMAIN false #endif @@ -59,8 +62,8 @@ #endif #include -#include #include +#include #include // for thread caching #include // for hash set in bulk reclamation @@ -69,16 +72,15 @@ namespace folly { namespace hazptr { /** - * helper classes and functions + * Helper classes and functions */ /** hazptr_stats */ class hazptr_stats; -hazptr_stats& hazptr_stats(); #if HAZPTR_STATS -#define INC_HAZPTR_STATS(x) hazptr_stats().x() +#define INC_HAZPTR_STATS(x) hazptr_stats_.x() #else #define INC_HAZPTR_STATS(x) #endif @@ -91,64 +93,101 @@ class hazptr_mb { static void heavy(); }; -/** hazptr_tc +/** + * TLS structures + */ + +/** TLS life state */ + +enum hazptr_tls_state { TLS_ALIVE, TLS_UNINITIALIZED, TLS_DESTROYED }; + +/** hazptr_tc structures * Thread caching of hazptr_rec-s that belong to the default domain. */ -class hazptr_tc_entry { - friend class hazptr_tc; - hazptr_rec* hprec_{nullptr}; +struct hazptr_tc_entry { + hazptr_rec* hprec_; - public: void fill(hazptr_rec* hprec); hazptr_rec* get(); void evict(); }; -class hazptr_tc { - hazptr_tc_entry tc_[HAZPTR_TC_SIZE]; - int count_{0}; +static_assert( + std::is_trivial::value, + "hazptr_tc_entry must be trivial" + " to avoid a branch to check initialization"); + +struct hazptr_tc { + hazptr_tc_entry entry_[HAZPTR_TC_SIZE]; + size_t count_; + bool local_; // for debug mode only public: - hazptr_tc(); - ~hazptr_tc(); + hazptr_tc_entry& operator[](size_t i); hazptr_rec* get(); bool put(hazptr_rec* hprec); + size_t count(); }; -hazptr_tc& hazptr_tc(); +static_assert( + std::is_trivial::value, + "hazptr_tc must be trivial to avoid a branch to check initialization"); + +hazptr_tc* hazptr_tc_tls(); +void hazptr_tc_init(); +void hazptr_tc_shutdown(); +hazptr_rec* hazptr_tc_try_get(); +bool hazptr_tc_try_put(hazptr_rec* hprec); -/** hazptr_priv +/** hazptr_priv structures * Thread private lists of retired objects that belong to the default domain. */ -class hazptr_priv { - hazptr_domain* domain_{&default_hazptr_domain()}; - hazptr_obj* head_{nullptr}; - hazptr_obj* tail_{nullptr}; - int rcount_{0}; +struct hazptr_priv { + hazptr_obj* head_; + hazptr_obj* tail_; + int rcount_; + bool active_; - public: - hazptr_priv(); - ~hazptr_priv(); void push(hazptr_obj* obj); - - private: void pushAllToDomain(); }; -hazptr_priv& hazptr_priv(); +static_assert( + std::is_trivial::value, + "hazptr_priv must be trivial to avoid a branch to check initialization"); + +void hazptr_priv_init(); +void hazptr_priv_shutdown(); +bool hazptr_priv_try_retire(hazptr_obj* obj); + +/** hazptr_tls_life */ + +struct hazptr_tls_life { + hazptr_tls_life(); + ~hazptr_tls_life(); +}; + +void tls_life_odr_use(); + +/** tls globals */ + +extern thread_local hazptr_tls_state tls_state_; +extern thread_local hazptr_tc tls_tc_data_; +extern thread_local hazptr_priv tls_priv_data_; +extern thread_local hazptr_tls_life tls_life_; // last /** - * public functions + * hazptr_domain */ -/** hazptr_domain */ - -constexpr hazptr_domain::hazptr_domain(memory_resource* mr) noexcept +inline constexpr hazptr_domain::hazptr_domain(memory_resource* mr) noexcept : mr_(mr) {} -/** hazptr_obj_base */ +/** + * hazptr_obj_base + */ template inline void hazptr_obj_base::retire(hazptr_domain& domain, D deleter) { @@ -161,20 +200,85 @@ inline void hazptr_obj_base::retire(hazptr_domain& domain, D deleter) { }; if (HAZPTR_PRIV && (HAZPTR_ONE_DOMAIN || (&domain == &default_hazptr_domain()))) { - hazptr_priv().push(this); + if (hazptr_priv_try_retire(this)) { + return; + } + } + domain.objRetire(this); +} + +/** + * hazptr_obj_base_refcounted + */ + +template +inline void hazptr_obj_base_refcounted::retire( + hazptr_domain& domain, + D deleter) { + DEBUG_PRINT(this << " " << &domain); + preRetire(deleter); + if (HAZPTR_PRIV && + (HAZPTR_ONE_DOMAIN || (&domain == &default_hazptr_domain()))) { + if (hazptr_priv_try_retire(this)) { + return; + } + } + domain.objRetire(this); +} + +template +inline void hazptr_obj_base_refcounted::acquire_ref() { + DEBUG_PRINT(this); + auto oldval = refcount_.fetch_add(1); + DCHECK(oldval >= 0); +} + +template +inline void hazptr_obj_base_refcounted::acquire_ref_safe() { + DEBUG_PRINT(this); + auto oldval = refcount_.load(std::memory_order_acquire); + DCHECK(oldval >= 0); + refcount_.store(oldval + 1, std::memory_order_release); +} + +template +inline bool hazptr_obj_base_refcounted::release_ref() { + DEBUG_PRINT(this); + auto oldval = refcount_.load(std::memory_order_acquire); + if (oldval > 0) { + oldval = refcount_.fetch_sub(1); } else { - domain.objRetire(this); + if (kIsDebug) { + refcount_.store(-1); + } } + DEBUG_PRINT(this << " " << oldval); + DCHECK(oldval >= 0); + return oldval == 0; } -/** hazptr_rec */ +template +inline void hazptr_obj_base_refcounted::preRetire(D deleter) { + DCHECK(next_ == nullptr); + deleter_ = std::move(deleter); + reclaim_ = [](hazptr_obj* p) { + auto hrobp = static_cast(p); + if (hrobp->release_ref()) { + auto obj = static_cast(hrobp); + hrobp->deleter_(obj); + } + }; +} + +/** + * hazptr_rec + */ -class hazptr_rec { +class alignas(hardware_destructive_interference_size) hazptr_rec { friend class hazptr_domain; friend class hazptr_holder; - friend class hazptr_tc_entry; + friend struct hazptr_tc_entry; - FOLLY_ALIGN_TO_AVOID_FALSE_SHARING std::atomic hazptr_{nullptr}; hazptr_rec* next_{nullptr}; std::atomic active_{false}; @@ -187,16 +291,28 @@ class hazptr_rec { void release() noexcept; }; -/** hazptr_holder */ +/** + * hazptr_holder + */ FOLLY_ALWAYS_INLINE hazptr_holder::hazptr_holder(hazptr_domain& domain) { domain_ = &domain; + if (LIKELY( + HAZPTR_TC && + (HAZPTR_ONE_DOMAIN || &domain == &default_hazptr_domain()))) { + auto hprec = hazptr_tc_try_get(); + if (LIKELY(hprec != nullptr)) { + hazptr_ = hprec; + DEBUG_PRINT(this << " " << domain_ << " " << hazptr_); + return; + } + } hazptr_ = domain_->hazptrAcquire(); DEBUG_PRINT(this << " " << domain_ << " " << hazptr_); if (hazptr_ == nullptr) { std::bad_alloc e; throw e; } } -inline hazptr_holder::hazptr_holder(std::nullptr_t) { +FOLLY_ALWAYS_INLINE hazptr_holder::hazptr_holder(std::nullptr_t) noexcept { domain_ = nullptr; hazptr_ = nullptr; DEBUG_PRINT(this << " " << domain_ << " " << hazptr_); @@ -204,22 +320,30 @@ inline hazptr_holder::hazptr_holder(std::nullptr_t) { FOLLY_ALWAYS_INLINE hazptr_holder::~hazptr_holder() { DEBUG_PRINT(this); - if (domain_) { + if (LIKELY(hazptr_ != nullptr)) { hazptr_->clear(); + if (LIKELY( + HAZPTR_TC && + (HAZPTR_ONE_DOMAIN || domain_ == &default_hazptr_domain()))) { + if (LIKELY(hazptr_tc_try_put(hazptr_))) { + return; + } + } domain_->hazptrRelease(hazptr_); } } -inline hazptr_holder::hazptr_holder(hazptr_holder&& rhs) noexcept { +FOLLY_ALWAYS_INLINE hazptr_holder::hazptr_holder(hazptr_holder&& rhs) noexcept { domain_ = rhs.domain_; hazptr_ = rhs.hazptr_; rhs.domain_ = nullptr; rhs.hazptr_ = nullptr; } -inline hazptr_holder& hazptr_holder::operator=(hazptr_holder&& rhs) noexcept { +FOLLY_ALWAYS_INLINE +hazptr_holder& hazptr_holder::operator=(hazptr_holder&& rhs) noexcept { /* Self-move is a no-op. */ - if (this != &rhs) { + if (LIKELY(this != &rhs)) { this->~hazptr_holder(); new (this) hazptr_holder(std::move(rhs)); } @@ -230,11 +354,19 @@ template FOLLY_ALWAYS_INLINE bool hazptr_holder::try_protect( T*& ptr, const std::atomic& src) noexcept { + return try_protect(ptr, src, [](T* t) { return t; }); +} + +template +FOLLY_ALWAYS_INLINE bool hazptr_holder::try_protect( + T*& ptr, + const std::atomic& src, + Func f) noexcept { DEBUG_PRINT(this << " " << ptr << " " << &src); - reset(ptr); + reset(f(ptr)); /*** Full fence ***/ hazptr_mb::light(); T* p = src.load(std::memory_order_acquire); - if (p != ptr) { + if (UNLIKELY(p != ptr)) { ptr = p; reset(); return false; @@ -245,8 +377,16 @@ FOLLY_ALWAYS_INLINE bool hazptr_holder::try_protect( template FOLLY_ALWAYS_INLINE T* hazptr_holder::get_protected( const std::atomic& src) noexcept { + return get_protected(src, [](T* t) { return t; }); +} + +template +FOLLY_ALWAYS_INLINE T* hazptr_holder::get_protected( + const std::atomic& src, + Func f) noexcept { T* p = src.load(std::memory_order_relaxed); - while (!try_protect(p, src)) {} + while (!try_protect(p, src, f)) { + } DEBUG_PRINT(this << " " << p << " " << &src); return p; } @@ -279,6 +419,186 @@ FOLLY_ALWAYS_INLINE void swap(hazptr_holder& lhs, hazptr_holder& rhs) noexcept { lhs.swap(rhs); } +/** + * hazptr_array + */ + +template +FOLLY_ALWAYS_INLINE hazptr_array::hazptr_array() { + auto h = reinterpret_cast(&raw_); + if (HAZPTR_TC) { + auto ptc = hazptr_tc_tls(); + if (LIKELY(ptc != nullptr)) { + auto& tc = *ptc; + auto count = tc.count(); + if (M <= count) { + size_t offset = count - M; + for (size_t i = 0; i < M; ++i) { + auto hprec = tc[offset + i].hprec_; + DCHECK(hprec != nullptr); + DEBUG_PRINT(i << " " << &h[i]); + new (&h[i]) hazptr_holder(nullptr); + h[i].hazptr_ = hprec; + DEBUG_PRINT( + i << " " << &h[i] << " " << h[i].domain_ << " " << h[i].hazptr_); + } + tc.count_ = offset; + return; + } + } + } + // slow path + for (size_t i = 0; i < M; ++i) { + new (&h[i]) hazptr_holder; + DEBUG_PRINT( + i << " " << &h[i] << " " << h[i].domain_ << " " << h[i].hazptr_); + } +} + +template +FOLLY_ALWAYS_INLINE hazptr_array::hazptr_array( + hazptr_array&& other) noexcept { + DEBUG_PRINT(this << " " << M << " " << &other); + auto h = reinterpret_cast(&raw_); + auto hother = reinterpret_cast(&other.raw_); + for (size_t i = 0; i < M; ++i) { + new (&h[i]) hazptr_holder(std::move(hother[i])); + DEBUG_PRINT(i << " " << &h[i] << " " << &hother[i]); + } + empty_ = other.empty_; + other.empty_ = true; +} + +template +FOLLY_ALWAYS_INLINE hazptr_array::hazptr_array(std::nullptr_t) noexcept { + DEBUG_PRINT(this << " " << M); + auto h = reinterpret_cast(&raw_); + for (size_t i = 0; i < M; ++i) { + new (&h[i]) hazptr_holder(nullptr); + DEBUG_PRINT(i << " " << &h[i]); + } + empty_ = true; +} + +template +FOLLY_ALWAYS_INLINE hazptr_array::~hazptr_array() { + if (empty_) { + return; + } + auto h = reinterpret_cast(&raw_); + if (HAZPTR_TC) { + auto ptc = hazptr_tc_tls(); + if (LIKELY(ptc != nullptr)) { + auto& tc = *ptc; + auto count = tc.count(); + if ((M <= HAZPTR_TC_SIZE) && (count + M <= HAZPTR_TC_SIZE)) { + for (size_t i = 0; i < M; ++i) { + tc[count + i].hprec_ = h[i].hazptr_; + DEBUG_PRINT(i << " " << &h[i]); + new (&h[i]) hazptr_holder(nullptr); + DEBUG_PRINT( + i << " " << &h[i] << " " << h[i].domain_ << " " << h[i].hazptr_); + } + tc.count_ = count + M; + return; + } + } + } + // slow path + for (size_t i = 0; i < M; ++i) { + h[i].~hazptr_holder(); + } +} + +template +FOLLY_ALWAYS_INLINE hazptr_array& hazptr_array::operator=( + hazptr_array&& other) noexcept { + DEBUG_PRINT(this << " " << M << " " << &other); + auto h = reinterpret_cast(&raw_); + for (size_t i = 0; i < M; ++i) { + h[i] = std::move(other[i]); + DEBUG_PRINT(i << " " << &h[i] << " " << &other[i]); + } + empty_ = other.empty_; + other.empty_ = true; + return *this; +} + +template +FOLLY_ALWAYS_INLINE hazptr_holder& hazptr_array::operator[]( + size_t i) noexcept { + auto h = reinterpret_cast(&raw_); + DCHECK(i < M); + return h[i]; +} + +/** + * hazptr_local + */ + +template +FOLLY_ALWAYS_INLINE hazptr_local::hazptr_local() { + auto h = reinterpret_cast(&raw_); + if (HAZPTR_TC) { + auto ptc = hazptr_tc_tls(); + if (LIKELY(ptc != nullptr)) { + auto& tc = *ptc; + auto count = tc.count(); + if (M <= count) { + if (kIsDebug) { + DCHECK(!tc.local_); + tc.local_ = true; + } + // Fast path + for (size_t i = 0; i < M; ++i) { + auto hprec = tc[i].hprec_; + DCHECK(hprec != nullptr); + DEBUG_PRINT(i << " " << &h[i]); + new (&h[i]) hazptr_holder(nullptr); + h[i].hazptr_ = hprec; + DEBUG_PRINT( + i << " " << &h[i] << " " << h[i].domain_ << " " << h[i].hazptr_); + } + return; + } + } + } + // Slow path + need_destruct_ = true; + for (size_t i = 0; i < M; ++i) { + new (&h[i]) hazptr_holder; + DEBUG_PRINT( + i << " " << &h[i] << " " << h[i].domain_ << " " << h[i].hazptr_); + } +} + +template +FOLLY_ALWAYS_INLINE hazptr_local::~hazptr_local() { + if (LIKELY(!need_destruct_)) { + if (kIsDebug) { + auto ptc = hazptr_tc_tls(); + DCHECK(ptc != nullptr); + auto& tc = *ptc; + DCHECK(tc.local_); + tc.local_ = false; + } + return; + } + // Slow path + auto h = reinterpret_cast(&raw_); + for (size_t i = 0; i < M; ++i) { + h[i].~hazptr_holder(); + } +} + +template +FOLLY_ALWAYS_INLINE hazptr_holder& hazptr_local::operator[]( + size_t i) noexcept { + auto h = reinterpret_cast(&raw_); + DCHECK(i < M); + return h[i]; +} + //////////////////////////////////////////////////////////////////////////////// // [TODO]: // - Control of reclamation (when and by whom) @@ -286,15 +606,19 @@ FOLLY_ALWAYS_INLINE void swap(hazptr_holder& lhs, hazptr_holder& rhs) noexcept { /** Definition of default_hazptr_domain() */ -inline hazptr_domain& default_hazptr_domain() { - static hazptr_domain d; - DEBUG_PRINT(&d); - return d; +FOLLY_ALWAYS_INLINE hazptr_domain& default_hazptr_domain() { + DEBUG_PRINT(&default_domain_); + return default_domain_; +} + +template +FOLLY_ALWAYS_INLINE void hazptr_retire(T* obj, D reclaim) { + default_hazptr_domain().retire(obj, std::move(reclaim)); } /** hazptr_rec */ -inline void hazptr_rec::set(const void* p) noexcept { +FOLLY_ALWAYS_INLINE void hazptr_rec::set(const void* p) noexcept { DEBUG_PRINT(this << " " << p); hazptr_.store(p, std::memory_order_release); } @@ -305,7 +629,7 @@ inline const void* hazptr_rec::get() const noexcept { return p; } -inline void hazptr_rec::clear() noexcept { +FOLLY_ALWAYS_INLINE void hazptr_rec::clear() noexcept { DEBUG_PRINT(this); hazptr_.store(nullptr, std::memory_order_release); } @@ -339,6 +663,21 @@ inline const void* hazptr_obj::getObjPtr() const { /** hazptr_domain */ +template +void hazptr_domain::retire(T* obj, D reclaim) { + struct hazptr_retire_node : hazptr_obj { + std::unique_ptr obj_; + + hazptr_retire_node(T* obj, D reclaim) : obj_{obj, std::move(reclaim)} {} + }; + + auto node = new hazptr_retire_node(obj, std::move(reclaim)); + node->reclaim_ = [](hazptr_obj* p) { + delete static_cast(p); + }; + objRetire(node); +} + inline hazptr_domain::~hazptr_domain() { DEBUG_PRINT(this); { /* reclaim all remaining retired objects */ @@ -347,12 +686,18 @@ inline hazptr_domain::~hazptr_domain() { while (retired) { for (auto p = retired; p; p = next) { next = p->next_; + DCHECK(p != next); + DEBUG_PRINT(this << " " << p << " " << p->reclaim_); (*(p->reclaim_))(p); } retired = retired_.exchange(nullptr); } } - { /* free all hazptr_rec-s */ + /* Leak the data for the default domain to avoid destruction order + * issues with thread caches. + */ + if (this != &default_hazptr_domain()) { + /* free all hazptr_rec-s */ hazptr_rec* next; for (auto p = hazptrs_.load(std::memory_order_acquire); p; p = next) { next = p->next_; @@ -362,13 +707,7 @@ inline hazptr_domain::~hazptr_domain() { } } -FOLLY_ALWAYS_INLINE hazptr_rec* hazptr_domain::hazptrAcquire() { - if (HAZPTR_TC && (HAZPTR_ONE_DOMAIN || this == &default_hazptr_domain())) { - auto hprec = hazptr_tc().get(); - if (hprec) { - return hprec; - } - } +inline hazptr_rec* hazptr_domain::hazptrAcquire() { hazptr_rec* p; hazptr_rec* next; for (p = hazptrs_.load(std::memory_order_acquire); p; p = next) { @@ -378,24 +717,22 @@ FOLLY_ALWAYS_INLINE hazptr_rec* hazptr_domain::hazptrAcquire() { } } p = static_cast(mr_->allocate(sizeof(hazptr_rec))); + DEBUG_PRINT(this << " " << p << " " << sizeof(hazptr_rec)); if (p == nullptr) { return nullptr; } p->active_.store(true, std::memory_order_relaxed); p->next_ = hazptrs_.load(std::memory_order_acquire); while (!hazptrs_.compare_exchange_weak( - p->next_, p, std::memory_order_release, std::memory_order_acquire)) + p->next_, p, std::memory_order_release, std::memory_order_acquire)) { /* keep trying */; + } auto hcount = hcount_.fetch_add(1); DEBUG_PRINT(this << " " << p << " " << sizeof(hazptr_rec) << " " << hcount); return p; } -FOLLY_ALWAYS_INLINE void hazptr_domain::hazptrRelease(hazptr_rec* p) noexcept { - if (HAZPTR_TC && (HAZPTR_ONE_DOMAIN || this == &default_hazptr_domain()) && - hazptr_tc().put(p)) { - return; - } +inline void hazptr_domain::hazptrRelease(hazptr_rec* p) noexcept { DEBUG_PRINT(this << " " << p); p->release(); } @@ -457,6 +794,7 @@ inline void hazptr_domain::bulkReclaim() { hazptr_obj* next; for (; p; p = next) { next = p->next_; + DCHECK(p != next); if (hs.count(p->getObjPtr()) == 0) { DEBUG_PRINT(this << " " << p << " " << p->reclaim_); (*(p->reclaim_))(p); @@ -489,6 +827,8 @@ class hazptr_stats { std::atomic seq_cst_{0}; }; +extern hazptr_stats hazptr_stats_; + inline hazptr_stats::~hazptr_stats() { DEBUG_PRINT(this << " light " << light_.load()); DEBUG_PRINT(this << " heavy " << heavy_.load()); @@ -513,15 +853,9 @@ inline void hazptr_stats::seq_cst() { } } -inline class hazptr_stats& hazptr_stats() { - static class hazptr_stats stats_; - DEBUG_PRINT(&stats_); - return stats_; -} - /** hazptr_mb */ -inline void hazptr_mb::light() { +FOLLY_ALWAYS_INLINE void hazptr_mb::light() { DEBUG_PRINT(""); if (HAZPTR_AMB) { folly::asymmetricLightBarrier(); @@ -543,41 +877,43 @@ inline void hazptr_mb::heavy() { } } -/** hazptr_tc - functions */ +/** + * TLS structures + */ + +/** + * hazptr_tc structures + */ -inline void hazptr_tc_entry::fill(hazptr_rec* hprec) { +/** hazptr_tc_entry */ + +FOLLY_ALWAYS_INLINE void hazptr_tc_entry::fill(hazptr_rec* hprec) { hprec_ = hprec; DEBUG_PRINT(this << " " << hprec); } -inline hazptr_rec* hazptr_tc_entry::get() { +FOLLY_ALWAYS_INLINE hazptr_rec* hazptr_tc_entry::get() { auto hprec = hprec_; - hprec_ = nullptr; DEBUG_PRINT(this << " " << hprec); return hprec; } inline void hazptr_tc_entry::evict() { auto hprec = hprec_; - hprec_ = nullptr; hprec->release(); DEBUG_PRINT(this << " " << hprec); } -inline hazptr_tc::hazptr_tc() { - DEBUG_PRINT(this); -} +/** hazptr_tc */ -inline hazptr_tc::~hazptr_tc() { - DEBUG_PRINT(this); - for (int i = 0; i < count_; ++i) { - tc_[i].evict(); - } +FOLLY_ALWAYS_INLINE hazptr_tc_entry& hazptr_tc::operator[](size_t i) { + DCHECK(i <= HAZPTR_TC_SIZE); + return entry_[i]; } -inline hazptr_rec* hazptr_tc::get() { - if (count_ > 0) { - auto hprec = tc_[--count_].get(); +FOLLY_ALWAYS_INLINE hazptr_rec* hazptr_tc::get() { + if (LIKELY(count_ != 0)) { + auto hprec = entry_[--count_].get(); DEBUG_PRINT(this << " " << hprec); return hprec; } @@ -585,61 +921,231 @@ inline hazptr_rec* hazptr_tc::get() { return nullptr; } -inline bool hazptr_tc::put(hazptr_rec* hprec) { - if (count_ < HAZPTR_TC_SIZE) { - tc_[count_++].fill(hprec); +FOLLY_ALWAYS_INLINE bool hazptr_tc::put(hazptr_rec* hprec) { + if (LIKELY(count_ < HAZPTR_TC_SIZE)) { + entry_[count_++].fill(hprec); DEBUG_PRINT(this << " " << count_ - 1); return true; } return false; } -inline class hazptr_tc& hazptr_tc() { - static thread_local class hazptr_tc tc; +FOLLY_ALWAYS_INLINE size_t hazptr_tc::count() { + return count_; +} + +/** hazptr_tc free functions */ + +FOLLY_ALWAYS_INLINE hazptr_tc* hazptr_tc_tls() { + DEBUG_PRINT(tls_state_); + if (LIKELY(tls_state_ == TLS_ALIVE)) { + DEBUG_PRINT(tls_state_); + return &tls_tc_data_; + } else if (tls_state_ == TLS_UNINITIALIZED) { + tls_life_odr_use(); + return &tls_tc_data_; + } + return nullptr; +} + +inline void hazptr_tc_init() { + DEBUG_PRINT(""); + auto& tc = tls_tc_data_; DEBUG_PRINT(&tc); - return tc; + tc.count_ = 0; + if (kIsDebug) { + tc.local_ = false; + } } -/** hazptr_priv - functions */ +inline void hazptr_tc_shutdown() { + auto& tc = tls_tc_data_; + DEBUG_PRINT(&tc); + for (size_t i = 0; i < tc.count_; ++i) { + tc.entry_[i].evict(); + } +} -inline hazptr_priv::hazptr_priv() { - DEBUG_PRINT(this); +FOLLY_ALWAYS_INLINE hazptr_rec* hazptr_tc_try_get() { + DEBUG_PRINT(TLS_UNINITIALIZED << TLS_ALIVE << TLS_DESTROYED); + DEBUG_PRINT(tls_state_); + if (LIKELY(tls_state_ == TLS_ALIVE)) { + DEBUG_PRINT(tls_state_); + return tls_tc_data_.get(); + } else if (tls_state_ == TLS_UNINITIALIZED) { + tls_life_odr_use(); + return tls_tc_data_.get(); + } + return nullptr; } -inline hazptr_priv::~hazptr_priv() { - DEBUG_PRINT(this); - if (tail_) { - pushAllToDomain(); +FOLLY_ALWAYS_INLINE bool hazptr_tc_try_put(hazptr_rec* hprec) { + DEBUG_PRINT(tls_state_); + if (LIKELY(tls_state_ == TLS_ALIVE)) { + DEBUG_PRINT(tls_state_); + return tls_tc_data_.put(hprec); } + return false; } +/** + * hazptr_priv + */ + inline void hazptr_priv::push(hazptr_obj* obj) { + auto& domain = default_hazptr_domain(); obj->next_ = nullptr; if (tail_) { tail_->next_ = obj; } else { + if (!active_) { + domain.objRetire(obj); + return; + } head_ = obj; } tail_ = obj; - ++rcount_; - if (domain_->reachedThreshold(rcount_)) { + if (++rcount_ >= HAZPTR_PRIV_THRESHOLD) { pushAllToDomain(); } } inline void hazptr_priv::pushAllToDomain() { - domain_->pushRetired(head_, tail_, rcount_); + auto& domain = default_hazptr_domain(); + domain.pushRetired(head_, tail_, rcount_); head_ = nullptr; tail_ = nullptr; rcount_ = 0; - domain_->tryBulkReclaim(); + domain.tryBulkReclaim(); } -inline class hazptr_priv& hazptr_priv() { - static thread_local class hazptr_priv priv; +inline void hazptr_priv_init() { + auto& priv = tls_priv_data_; DEBUG_PRINT(&priv); - return priv; + priv.head_ = nullptr; + priv.tail_ = nullptr; + priv.rcount_ = 0; + priv.active_ = true; } -} // namespace folly +inline void hazptr_priv_shutdown() { + auto& priv = tls_priv_data_; + DEBUG_PRINT(&priv); + DCHECK(priv.active_); + priv.active_ = false; + if (priv.tail_) { + priv.pushAllToDomain(); + } +} + +inline bool hazptr_priv_try_retire(hazptr_obj* obj) { + DEBUG_PRINT(tls_state_); + if (tls_state_ == TLS_ALIVE) { + DEBUG_PRINT(tls_state_); + tls_priv_data_.push(obj); + return true; + } else if (tls_state_ == TLS_UNINITIALIZED) { + DEBUG_PRINT(tls_state_); + tls_life_odr_use(); + tls_priv_data_.push(obj); + return true; + } + return false; +} + +/** hazptr_tls_life */ + +inline void tls_life_odr_use() { + DEBUG_PRINT(tls_state_); + CHECK(tls_state_ == TLS_UNINITIALIZED); + auto volatile tlsOdrUse = &tls_life_; + CHECK(tlsOdrUse != nullptr); + DEBUG_PRINT(tlsOdrUse); +} + +inline hazptr_tls_life::hazptr_tls_life() { + DEBUG_PRINT(this); + CHECK(tls_state_ == TLS_UNINITIALIZED); + hazptr_tc_init(); + hazptr_priv_init(); + tls_state_ = TLS_ALIVE; +} + +inline hazptr_tls_life::~hazptr_tls_life() { + DEBUG_PRINT(this); + CHECK(tls_state_ == TLS_ALIVE); + hazptr_tc_shutdown(); + hazptr_priv_shutdown(); + tls_state_ = TLS_DESTROYED; +} + +/** hazptr_obj_batch */ +/* Only for default domain. Supports only hazptr_obj_base_refcounted + * and a thread-safe access only, for now. */ + +class hazptr_obj_batch { + static constexpr size_t DefaultThreshold = 20; + hazptr_obj* head_{nullptr}; + hazptr_obj* tail_{nullptr}; + size_t rcount_{0}; + size_t threshold_{DefaultThreshold}; + + public: + hazptr_obj_batch() {} + hazptr_obj_batch(hazptr_obj* head, hazptr_obj* tail, size_t rcount) + : head_(head), tail_(tail), rcount_(rcount) {} + + ~hazptr_obj_batch() { + retire_all(); + } + + /* Prepare a hazptr_obj_base_refcounted for retirement but don't + push it the domain yet. Return true if the batch is ready. */ + template > + hazptr_obj_batch prep_retire_refcounted( + hazptr_obj_base_refcounted* obj, + D deleter = {}) { + obj->preRetire(deleter); + obj->next_ = head_; + head_ = obj; + if (tail_ == nullptr) { + tail_ = obj; + } + if (++rcount_ < threshold_) { + return hazptr_obj_batch(); + } else { + auto head = head_; + auto tail = tail_; + auto rcount = rcount_; + clear(); + return hazptr_obj_batch(head, tail, rcount); + } + } + + bool empty() { + return rcount_ == 0; + } + + void retire_all() { + if (!empty()) { + auto& domain = default_hazptr_domain(); + domain.pushRetired(head_, tail_, rcount_); + domain.tryBulkReclaim(); + clear(); + } + } + + void set_threshold(size_t thresh) { + threshold_ = thresh; + } + + private: + void clear() { + head_ = nullptr; + tail_ = nullptr; + rcount_ = 0; + } +}; + } // namespace hazptr +} // namespace folly