2 * Copyright 2017 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 /* override-include-guard */
19 #error "This should only be included by hazptr.h"
22 /* quality of implementation switches */
24 // NOTE: The #ifndef pattern is prone to ODR violation. Its use for
25 // quality of implementation options is temporary. Eventually these
26 // options should be added to the API in future API extensions.
29 #define HAZPTR_AMB true
33 #define HAZPTR_TC true
36 #ifndef HAZPTR_TC_SIZE
37 #define HAZPTR_TC_SIZE 10
41 #define HAZPTR_PRIV true
44 #ifndef HAZPTR_ONE_DOMAIN
45 #define HAZPTR_ONE_DOMAIN false
48 #ifndef HAZPTR_SCAN_MULT
49 #define HAZPTR_SCAN_MULT 2
52 #ifndef HAZPTR_SCAN_THRESHOLD
53 #define HAZPTR_SCAN_THRESHOLD 1000
58 #define HAZPTR_STATS false
61 #include <folly/concurrency/CacheLocality.h>
62 #include <folly/experimental/hazptr/debug.h>
63 #include <folly/synchronization/AsymmetricMemoryBarrier.h>
65 #include <mutex> // for thread caching
66 #include <unordered_set> // for hash set in bulk reclamation
72 * Helper classes and functions
80 #define INC_HAZPTR_STATS(x) hazptr_stats_.x()
82 #define INC_HAZPTR_STATS(x)
99 enum hazptr_tls_state { TLS_ALIVE, TLS_UNINITIALIZED, TLS_DESTROYED };
101 /** hazptr_tc structures
102 * Thread caching of hazptr_rec-s that belong to the default domain.
105 struct hazptr_tc_entry {
108 void fill(hazptr_rec* hprec);
114 std::is_trivial<hazptr_tc_entry>::value,
115 "hazptr_tc_entry must be trivial"
116 " to avoid a branch to check initialization");
119 hazptr_tc_entry entry_[HAZPTR_TC_SIZE];
121 bool local_; // for debug mode only
124 hazptr_tc_entry& operator[](size_t i);
126 bool put(hazptr_rec* hprec);
131 std::is_trivial<hazptr_tc>::value,
132 "hazptr_tc must be trivial to avoid a branch to check initialization");
134 hazptr_tc* hazptr_tc_tls();
135 void hazptr_tc_init();
136 void hazptr_tc_shutdown();
137 hazptr_rec* hazptr_tc_try_get();
138 bool hazptr_tc_try_put(hazptr_rec* hprec);
140 /** hazptr_priv structures
141 * Thread private lists of retired objects that belong to the default domain.
150 void push(hazptr_obj* obj);
151 void pushAllToDomain();
155 std::is_trivial<hazptr_priv>::value,
156 "hazptr_priv must be trivial to avoid a branch to check initialization");
158 void hazptr_priv_init();
159 void hazptr_priv_shutdown();
160 bool hazptr_priv_try_retire(hazptr_obj* obj);
162 /** hazptr_tls_life */
164 struct hazptr_tls_life {
169 void tls_life_odr_use();
173 extern thread_local hazptr_tls_state tls_state_;
174 extern thread_local hazptr_tc tls_tc_data_;
175 extern thread_local hazptr_priv tls_priv_data_;
176 extern thread_local hazptr_tls_life tls_life_; // last
182 inline constexpr hazptr_domain::hazptr_domain(memory_resource* mr) noexcept
189 template <typename T, typename D>
190 inline void hazptr_obj_base<T, D>::retire(hazptr_domain& domain, D deleter) {
191 DEBUG_PRINT(this << " " << &domain);
192 deleter_ = std::move(deleter);
193 reclaim_ = [](hazptr_obj* p) {
194 auto hobp = static_cast<hazptr_obj_base*>(p);
195 auto obj = static_cast<T*>(hobp);
199 (HAZPTR_ONE_DOMAIN || (&domain == &default_hazptr_domain()))) {
200 if (hazptr_priv_try_retire(this)) {
204 domain.objRetire(this);
208 * hazptr_obj_base_refcounted
211 template <typename T, typename D>
212 inline void hazptr_obj_base_refcounted<T, D>::retire(
213 hazptr_domain& domain,
215 DEBUG_PRINT(this << " " << &domain);
216 deleter_ = std::move(deleter);
217 reclaim_ = [](hazptr_obj* p) {
218 auto hrobp = static_cast<hazptr_obj_base_refcounted*>(p);
219 if (hrobp->release_ref()) {
220 auto obj = static_cast<T*>(hrobp);
221 hrobp->deleter_(obj);
225 (HAZPTR_ONE_DOMAIN || (&domain == &default_hazptr_domain()))) {
226 if (hazptr_priv_try_retire(this)) {
230 domain.objRetire(this);
233 template <typename T, typename D>
234 inline void hazptr_obj_base_refcounted<T, D>::acquire_ref() {
236 auto oldval = refcount_.fetch_add(1);
240 template <typename T, typename D>
241 inline void hazptr_obj_base_refcounted<T, D>::acquire_ref_safe() {
243 auto oldval = refcount_.load(std::memory_order_acquire);
245 refcount_.store(oldval + 1, std::memory_order_release);
248 template <typename T, typename D>
249 inline bool hazptr_obj_base_refcounted<T, D>::release_ref() {
251 auto oldval = refcount_.load(std::memory_order_acquire);
253 oldval = refcount_.fetch_sub(1);
259 DEBUG_PRINT(this << " " << oldval);
269 friend class hazptr_domain;
270 friend class hazptr_holder;
271 friend struct hazptr_tc_entry;
273 FOLLY_ALIGN_TO_AVOID_FALSE_SHARING
274 std::atomic<const void*> hazptr_{nullptr};
275 hazptr_rec* next_{nullptr};
276 std::atomic<bool> active_{false};
278 void set(const void* p) noexcept;
279 const void* get() const noexcept;
280 void clear() noexcept;
281 bool isActive() noexcept;
282 bool tryAcquire() noexcept;
283 void release() noexcept;
290 FOLLY_ALWAYS_INLINE hazptr_holder::hazptr_holder(hazptr_domain& domain) {
294 (HAZPTR_ONE_DOMAIN || &domain == &default_hazptr_domain()))) {
295 auto hprec = hazptr_tc_try_get();
296 if (LIKELY(hprec != nullptr)) {
298 DEBUG_PRINT(this << " " << domain_ << " " << hazptr_);
302 hazptr_ = domain_->hazptrAcquire();
303 DEBUG_PRINT(this << " " << domain_ << " " << hazptr_);
304 if (hazptr_ == nullptr) { std::bad_alloc e; throw e; }
307 FOLLY_ALWAYS_INLINE hazptr_holder::hazptr_holder(std::nullptr_t) noexcept {
310 DEBUG_PRINT(this << " " << domain_ << " " << hazptr_);
313 FOLLY_ALWAYS_INLINE hazptr_holder::~hazptr_holder() {
315 if (LIKELY(hazptr_ != nullptr)) {
319 (HAZPTR_ONE_DOMAIN || domain_ == &default_hazptr_domain()))) {
320 if (LIKELY(hazptr_tc_try_put(hazptr_))) {
324 domain_->hazptrRelease(hazptr_);
328 FOLLY_ALWAYS_INLINE hazptr_holder::hazptr_holder(hazptr_holder&& rhs) noexcept {
329 domain_ = rhs.domain_;
330 hazptr_ = rhs.hazptr_;
331 rhs.domain_ = nullptr;
332 rhs.hazptr_ = nullptr;
336 hazptr_holder& hazptr_holder::operator=(hazptr_holder&& rhs) noexcept {
337 /* Self-move is a no-op. */
338 if (LIKELY(this != &rhs)) {
339 this->~hazptr_holder();
340 new (this) hazptr_holder(std::move(rhs));
345 template <typename T>
346 FOLLY_ALWAYS_INLINE bool hazptr_holder::try_protect(
348 const std::atomic<T*>& src) noexcept {
349 return try_protect(ptr, src, [](T* t) { return t; });
352 template <typename T, typename Func>
353 FOLLY_ALWAYS_INLINE bool hazptr_holder::try_protect(
355 const std::atomic<T*>& src,
357 DEBUG_PRINT(this << " " << ptr << " " << &src);
359 /*** Full fence ***/ hazptr_mb::light();
360 T* p = src.load(std::memory_order_acquire);
361 if (UNLIKELY(p != ptr)) {
369 template <typename T>
370 FOLLY_ALWAYS_INLINE T* hazptr_holder::get_protected(
371 const std::atomic<T*>& src) noexcept {
372 return get_protected(src, [](T* t) { return t; });
375 template <typename T, typename Func>
376 FOLLY_ALWAYS_INLINE T* hazptr_holder::get_protected(
377 const std::atomic<T*>& src,
379 T* p = src.load(std::memory_order_relaxed);
380 while (!try_protect(p, src, f)) {
382 DEBUG_PRINT(this << " " << p << " " << &src);
386 template <typename T>
387 FOLLY_ALWAYS_INLINE void hazptr_holder::reset(const T* ptr) noexcept {
388 auto p = static_cast<hazptr_obj*>(const_cast<T*>(ptr));
389 DEBUG_PRINT(this << " " << ptr << " p:" << p);
390 DCHECK(hazptr_); // UB if *this is empty
394 FOLLY_ALWAYS_INLINE void hazptr_holder::reset(std::nullptr_t) noexcept {
396 DCHECK(hazptr_); // UB if *this is empty
400 FOLLY_ALWAYS_INLINE void hazptr_holder::swap(hazptr_holder& rhs) noexcept {
402 this << " " << this->hazptr_ << " " << this->domain_ << " -- "
403 << &rhs << " " << rhs.hazptr_ << " " << rhs.domain_);
404 if (!HAZPTR_ONE_DOMAIN) {
405 std::swap(this->domain_, rhs.domain_);
407 std::swap(this->hazptr_, rhs.hazptr_);
410 FOLLY_ALWAYS_INLINE void swap(hazptr_holder& lhs, hazptr_holder& rhs) noexcept {
419 FOLLY_ALWAYS_INLINE hazptr_array<M>::hazptr_array() {
420 auto h = reinterpret_cast<hazptr_holder*>(&raw_);
422 auto ptc = hazptr_tc_tls();
423 if (LIKELY(ptc != nullptr)) {
425 auto count = tc.count();
427 size_t offset = count - M;
428 for (size_t i = 0; i < M; ++i) {
429 auto hprec = tc[offset + i].hprec_;
430 DCHECK(hprec != nullptr);
431 DEBUG_PRINT(i << " " << &h[i]);
432 new (&h[i]) hazptr_holder(nullptr);
433 h[i].hazptr_ = hprec;
435 i << " " << &h[i] << " " << h[i].domain_ << " " << h[i].hazptr_);
443 for (size_t i = 0; i < M; ++i) {
444 new (&h[i]) hazptr_holder;
446 i << " " << &h[i] << " " << h[i].domain_ << " " << h[i].hazptr_);
451 FOLLY_ALWAYS_INLINE hazptr_array<M>::hazptr_array(
452 hazptr_array&& other) noexcept {
453 DEBUG_PRINT(this << " " << M << " " << &other);
454 auto h = reinterpret_cast<hazptr_holder*>(&raw_);
455 auto hother = reinterpret_cast<hazptr_holder*>(&other.raw_);
456 for (size_t i = 0; i < M; ++i) {
457 new (&h[i]) hazptr_holder(std::move(hother[i]));
458 DEBUG_PRINT(i << " " << &h[i] << " " << &hother[i]);
460 empty_ = other.empty_;
465 FOLLY_ALWAYS_INLINE hazptr_array<M>::hazptr_array(std::nullptr_t) noexcept {
466 DEBUG_PRINT(this << " " << M);
467 auto h = reinterpret_cast<hazptr_holder*>(&raw_);
468 for (size_t i = 0; i < M; ++i) {
469 new (&h[i]) hazptr_holder(nullptr);
470 DEBUG_PRINT(i << " " << &h[i]);
476 FOLLY_ALWAYS_INLINE hazptr_array<M>::~hazptr_array() {
480 auto h = reinterpret_cast<hazptr_holder*>(&raw_);
482 auto ptc = hazptr_tc_tls();
483 if (LIKELY(ptc != nullptr)) {
485 auto count = tc.count();
486 if ((M <= HAZPTR_TC_SIZE) && (count + M <= HAZPTR_TC_SIZE)) {
487 for (size_t i = 0; i < M; ++i) {
488 tc[count + i].hprec_ = h[i].hazptr_;
489 DEBUG_PRINT(i << " " << &h[i]);
490 new (&h[i]) hazptr_holder(nullptr);
492 i << " " << &h[i] << " " << h[i].domain_ << " " << h[i].hazptr_);
494 tc.count_ = count + M;
500 for (size_t i = 0; i < M; ++i) {
501 h[i].~hazptr_holder();
506 FOLLY_ALWAYS_INLINE hazptr_array<M>& hazptr_array<M>::operator=(
507 hazptr_array&& other) noexcept {
508 DEBUG_PRINT(this << " " << M << " " << &other);
509 auto h = reinterpret_cast<hazptr_holder*>(&raw_);
510 for (size_t i = 0; i < M; ++i) {
511 h[i] = std::move(other[i]);
512 DEBUG_PRINT(i << " " << &h[i] << " " << &other[i]);
514 empty_ = other.empty_;
520 FOLLY_ALWAYS_INLINE hazptr_holder& hazptr_array<M>::operator[](
522 auto h = reinterpret_cast<hazptr_holder*>(&raw_);
532 FOLLY_ALWAYS_INLINE hazptr_local<M>::hazptr_local() {
533 auto h = reinterpret_cast<hazptr_holder*>(&raw_);
535 auto ptc = hazptr_tc_tls();
536 if (LIKELY(ptc != nullptr)) {
538 auto count = tc.count();
545 for (size_t i = 0; i < M; ++i) {
546 auto hprec = tc[i].hprec_;
547 DCHECK(hprec != nullptr);
548 DEBUG_PRINT(i << " " << &h[i]);
549 new (&h[i]) hazptr_holder(nullptr);
550 h[i].hazptr_ = hprec;
552 i << " " << &h[i] << " " << h[i].domain_ << " " << h[i].hazptr_);
559 need_destruct_ = true;
560 for (size_t i = 0; i < M; ++i) {
561 new (&h[i]) hazptr_holder;
563 i << " " << &h[i] << " " << h[i].domain_ << " " << h[i].hazptr_);
568 FOLLY_ALWAYS_INLINE hazptr_local<M>::~hazptr_local() {
569 if (LIKELY(!need_destruct_)) {
571 auto ptc = hazptr_tc_tls();
572 DCHECK(ptc != nullptr);
580 auto h = reinterpret_cast<hazptr_holder*>(&raw_);
581 for (size_t i = 0; i < M; ++i) {
582 h[i].~hazptr_holder();
587 FOLLY_ALWAYS_INLINE hazptr_holder& hazptr_local<M>::operator[](
589 auto h = reinterpret_cast<hazptr_holder*>(&raw_);
594 ////////////////////////////////////////////////////////////////////////////////
596 // - Control of reclamation (when and by whom)
597 // - End-to-end lock-free implementation
599 /** Definition of default_hazptr_domain() */
601 FOLLY_ALWAYS_INLINE hazptr_domain& default_hazptr_domain() {
602 DEBUG_PRINT(&default_domain_);
603 return default_domain_;
606 template <typename T, typename D>
607 FOLLY_ALWAYS_INLINE void hazptr_retire(T* obj, D reclaim) {
608 default_hazptr_domain().retire(obj, std::move(reclaim));
613 FOLLY_ALWAYS_INLINE void hazptr_rec::set(const void* p) noexcept {
614 DEBUG_PRINT(this << " " << p);
615 hazptr_.store(p, std::memory_order_release);
618 inline const void* hazptr_rec::get() const noexcept {
619 auto p = hazptr_.load(std::memory_order_acquire);
620 DEBUG_PRINT(this << " " << p);
624 FOLLY_ALWAYS_INLINE void hazptr_rec::clear() noexcept {
626 hazptr_.store(nullptr, std::memory_order_release);
629 inline bool hazptr_rec::isActive() noexcept {
630 return active_.load(std::memory_order_acquire);
633 inline bool hazptr_rec::tryAcquire() noexcept {
634 bool active = isActive();
636 active_.compare_exchange_strong(
637 active, true, std::memory_order_release, std::memory_order_relaxed)) {
644 inline void hazptr_rec::release() noexcept {
646 active_.store(false, std::memory_order_release);
651 inline const void* hazptr_obj::getObjPtr() const {
658 template <typename T, typename D>
659 void hazptr_domain::retire(T* obj, D reclaim) {
660 struct hazptr_retire_node : hazptr_obj {
661 std::unique_ptr<T, D> obj_;
663 hazptr_retire_node(T* obj, D reclaim) : obj_{obj, std::move(reclaim)} {}
666 auto node = new hazptr_retire_node(obj, std::move(reclaim));
667 node->reclaim_ = [](hazptr_obj* p) {
668 delete static_cast<hazptr_retire_node*>(p);
673 inline hazptr_domain::~hazptr_domain() {
675 { /* reclaim all remaining retired objects */
677 auto retired = retired_.exchange(nullptr);
679 for (auto p = retired; p; p = next) {
681 DEBUG_PRINT(this << " " << p << " " << p->reclaim_);
684 retired = retired_.exchange(nullptr);
687 /* Leak the data for the default domain to avoid destruction order
688 * issues with thread caches.
690 if (this != &default_hazptr_domain()) {
691 /* free all hazptr_rec-s */
693 for (auto p = hazptrs_.load(std::memory_order_acquire); p; p = next) {
695 DCHECK(!p->isActive());
696 mr_->deallocate(static_cast<void*>(p), sizeof(hazptr_rec));
701 inline hazptr_rec* hazptr_domain::hazptrAcquire() {
704 for (p = hazptrs_.load(std::memory_order_acquire); p; p = next) {
706 if (p->tryAcquire()) {
710 p = static_cast<hazptr_rec*>(mr_->allocate(sizeof(hazptr_rec)));
711 DEBUG_PRINT(this << " " << p << " " << sizeof(hazptr_rec));
715 p->active_.store(true, std::memory_order_relaxed);
716 p->next_ = hazptrs_.load(std::memory_order_acquire);
717 while (!hazptrs_.compare_exchange_weak(
718 p->next_, p, std::memory_order_release, std::memory_order_acquire)) {
721 auto hcount = hcount_.fetch_add(1);
722 DEBUG_PRINT(this << " " << p << " " << sizeof(hazptr_rec) << " " << hcount);
726 inline void hazptr_domain::hazptrRelease(hazptr_rec* p) noexcept {
727 DEBUG_PRINT(this << " " << p);
732 hazptr_domain::pushRetired(hazptr_obj* head, hazptr_obj* tail, int count) {
733 /*** Full fence ***/ hazptr_mb::light();
734 tail->next_ = retired_.load(std::memory_order_acquire);
735 while (!retired_.compare_exchange_weak(
738 std::memory_order_release,
739 std::memory_order_acquire)) {
741 return rcount_.fetch_add(count) + count;
744 inline bool hazptr_domain::reachedThreshold(int rcount) {
746 rcount >= HAZPTR_SCAN_THRESHOLD &&
747 rcount >= HAZPTR_SCAN_MULT * hcount_.load(std::memory_order_acquire));
750 inline void hazptr_domain::objRetire(hazptr_obj* p) {
751 auto rcount = pushRetired(p, p, 1);
752 if (reachedThreshold(rcount)) {
757 inline void hazptr_domain::tryBulkReclaim() {
760 auto hcount = hcount_.load(std::memory_order_acquire);
761 auto rcount = rcount_.load(std::memory_order_acquire);
762 if (rcount < HAZPTR_SCAN_THRESHOLD || rcount < HAZPTR_SCAN_MULT * hcount) {
765 if (rcount_.compare_exchange_weak(
766 rcount, 0, std::memory_order_release, std::memory_order_relaxed)) {
773 inline void hazptr_domain::bulkReclaim() {
775 /*** Full fence ***/ hazptr_mb::heavy();
776 auto p = retired_.exchange(nullptr, std::memory_order_acquire);
777 auto h = hazptrs_.load(std::memory_order_acquire);
778 std::unordered_set<const void*> hs; // TODO lock-free alternative
779 for (; h; h = h->next_) {
783 hazptr_obj* retired = nullptr;
784 hazptr_obj* tail = nullptr;
786 for (; p; p = next) {
788 if (hs.count(p->getObjPtr()) == 0) {
789 DEBUG_PRINT(this << " " << p << " " << p->reclaim_);
794 if (tail == nullptr) {
801 pushRetired(retired, tail, rcount);
815 std::atomic<uint64_t> light_{0};
816 std::atomic<uint64_t> heavy_{0};
817 std::atomic<uint64_t> seq_cst_{0};
820 extern hazptr_stats hazptr_stats_;
822 inline hazptr_stats::~hazptr_stats() {
823 DEBUG_PRINT(this << " light " << light_.load());
824 DEBUG_PRINT(this << " heavy " << heavy_.load());
825 DEBUG_PRINT(this << " seq_cst " << seq_cst_.load());
828 FOLLY_ALWAYS_INLINE void hazptr_stats::light() {
830 /* atomic */ ++light_;
834 inline void hazptr_stats::heavy() {
836 /* atomic */ ++heavy_;
840 inline void hazptr_stats::seq_cst() {
842 /* atomic */ ++seq_cst_;
848 FOLLY_ALWAYS_INLINE void hazptr_mb::light() {
851 folly::asymmetricLightBarrier();
852 INC_HAZPTR_STATS(light);
854 atomic_thread_fence(std::memory_order_seq_cst);
855 INC_HAZPTR_STATS(seq_cst);
859 inline void hazptr_mb::heavy() {
862 folly::asymmetricHeavyBarrier(AMBFlags::EXPEDITED);
863 INC_HAZPTR_STATS(heavy);
865 atomic_thread_fence(std::memory_order_seq_cst);
866 INC_HAZPTR_STATS(seq_cst);
875 * hazptr_tc structures
878 /** hazptr_tc_entry */
880 FOLLY_ALWAYS_INLINE void hazptr_tc_entry::fill(hazptr_rec* hprec) {
882 DEBUG_PRINT(this << " " << hprec);
885 FOLLY_ALWAYS_INLINE hazptr_rec* hazptr_tc_entry::get() {
887 DEBUG_PRINT(this << " " << hprec);
891 inline void hazptr_tc_entry::evict() {
894 DEBUG_PRINT(this << " " << hprec);
899 FOLLY_ALWAYS_INLINE hazptr_tc_entry& hazptr_tc::operator[](size_t i) {
900 DCHECK(i <= HAZPTR_TC_SIZE);
904 FOLLY_ALWAYS_INLINE hazptr_rec* hazptr_tc::get() {
905 if (LIKELY(count_ != 0)) {
906 auto hprec = entry_[--count_].get();
907 DEBUG_PRINT(this << " " << hprec);
910 DEBUG_PRINT(this << " nullptr");
914 FOLLY_ALWAYS_INLINE bool hazptr_tc::put(hazptr_rec* hprec) {
915 if (LIKELY(count_ < HAZPTR_TC_SIZE)) {
916 entry_[count_++].fill(hprec);
917 DEBUG_PRINT(this << " " << count_ - 1);
923 FOLLY_ALWAYS_INLINE size_t hazptr_tc::count() {
927 /** hazptr_tc free functions */
929 FOLLY_ALWAYS_INLINE hazptr_tc* hazptr_tc_tls() {
930 DEBUG_PRINT(tls_state_);
931 if (LIKELY(tls_state_ == TLS_ALIVE)) {
932 DEBUG_PRINT(tls_state_);
933 return &tls_tc_data_;
934 } else if (tls_state_ == TLS_UNINITIALIZED) {
936 return &tls_tc_data_;
941 inline void hazptr_tc_init() {
943 auto& tc = tls_tc_data_;
951 inline void hazptr_tc_shutdown() {
952 auto& tc = tls_tc_data_;
954 for (size_t i = 0; i < tc.count_; ++i) {
955 tc.entry_[i].evict();
959 FOLLY_ALWAYS_INLINE hazptr_rec* hazptr_tc_try_get() {
960 DEBUG_PRINT(TLS_UNINITIALIZED << TLS_ALIVE << TLS_DESTROYED);
961 DEBUG_PRINT(tls_state_);
962 if (LIKELY(tls_state_ == TLS_ALIVE)) {
963 DEBUG_PRINT(tls_state_);
964 return tls_tc_data_.get();
965 } else if (tls_state_ == TLS_UNINITIALIZED) {
967 return tls_tc_data_.get();
972 FOLLY_ALWAYS_INLINE bool hazptr_tc_try_put(hazptr_rec* hprec) {
973 DEBUG_PRINT(tls_state_);
974 if (LIKELY(tls_state_ == TLS_ALIVE)) {
975 DEBUG_PRINT(tls_state_);
976 return tls_tc_data_.put(hprec);
985 inline void hazptr_priv::push(hazptr_obj* obj) {
986 auto& domain = default_hazptr_domain();
987 obj->next_ = nullptr;
992 domain.objRetire(obj);
999 if (domain.reachedThreshold(rcount_)) {
1004 inline void hazptr_priv::pushAllToDomain() {
1005 auto& domain = default_hazptr_domain();
1006 domain.pushRetired(head_, tail_, rcount_);
1010 domain.tryBulkReclaim();
1013 inline void hazptr_priv_init() {
1014 auto& priv = tls_priv_data_;
1016 priv.head_ = nullptr;
1017 priv.tail_ = nullptr;
1019 priv.active_ = true;
1022 inline void hazptr_priv_shutdown() {
1023 auto& priv = tls_priv_data_;
1025 DCHECK(priv.active_);
1026 priv.active_ = false;
1028 priv.pushAllToDomain();
1032 inline bool hazptr_priv_try_retire(hazptr_obj* obj) {
1033 DEBUG_PRINT(tls_state_);
1034 if (tls_state_ == TLS_ALIVE) {
1035 DEBUG_PRINT(tls_state_);
1036 tls_priv_data_.push(obj);
1038 } else if (tls_state_ == TLS_UNINITIALIZED) {
1039 DEBUG_PRINT(tls_state_);
1041 tls_priv_data_.push(obj);
1047 /** hazptr_tls_life */
1049 inline void tls_life_odr_use() {
1050 DEBUG_PRINT(tls_state_);
1051 CHECK(tls_state_ == TLS_UNINITIALIZED);
1052 auto volatile tlsOdrUse = &tls_life_;
1053 CHECK(tlsOdrUse != nullptr);
1054 DEBUG_PRINT(tlsOdrUse);
1057 inline hazptr_tls_life::hazptr_tls_life() {
1059 CHECK(tls_state_ == TLS_UNINITIALIZED);
1062 tls_state_ = TLS_ALIVE;
1065 inline hazptr_tls_life::~hazptr_tls_life() {
1067 CHECK(tls_state_ == TLS_ALIVE);
1068 hazptr_tc_shutdown();
1069 hazptr_priv_shutdown();
1070 tls_state_ = TLS_DESTROYED;
1073 } // namespace hazptr
1074 } // namespace folly