+/** hazptr_stats */
+
+class hazptr_stats {
+ public:
+ ~hazptr_stats();
+ void light();
+ void heavy();
+ void seq_cst();
+
+ private:
+ std::atomic<uint64_t> light_{0};
+ std::atomic<uint64_t> heavy_{0};
+ std::atomic<uint64_t> seq_cst_{0};
+};
+
+extern hazptr_stats hazptr_stats_;
+
+inline hazptr_stats::~hazptr_stats() {
+ DEBUG_PRINT(this << " light " << light_.load());
+ DEBUG_PRINT(this << " heavy " << heavy_.load());
+ DEBUG_PRINT(this << " seq_cst " << seq_cst_.load());
+}
+
+FOLLY_ALWAYS_INLINE void hazptr_stats::light() {
+ if (HAZPTR_STATS) {
+ /* atomic */ ++light_;
+ }
+}
+
+inline void hazptr_stats::heavy() {
+ if (HAZPTR_STATS) {
+ /* atomic */ ++heavy_;
+ }
+}
+
+inline void hazptr_stats::seq_cst() {
+ if (HAZPTR_STATS) {
+ /* atomic */ ++seq_cst_;
+ }
+}
+
+/** hazptr_mb */
+
+FOLLY_ALWAYS_INLINE void hazptr_mb::light() {
+ DEBUG_PRINT("");
+ if (HAZPTR_AMB) {
+ folly::asymmetricLightBarrier();
+ INC_HAZPTR_STATS(light);
+ } else {
+ atomic_thread_fence(std::memory_order_seq_cst);
+ INC_HAZPTR_STATS(seq_cst);
+ }
+}
+
+inline void hazptr_mb::heavy() {
+ DEBUG_PRINT("");
+ if (HAZPTR_AMB) {
+ folly::asymmetricHeavyBarrier(AMBFlags::EXPEDITED);
+ INC_HAZPTR_STATS(heavy);
+ } else {
+ atomic_thread_fence(std::memory_order_seq_cst);
+ INC_HAZPTR_STATS(seq_cst);
+ }
+}
+
+/**
+ * TLS structures
+ */
+
+/**
+ * hazptr_tc structures
+ */
+
+/** hazptr_tc_entry */
+
+FOLLY_ALWAYS_INLINE void hazptr_tc_entry::fill(hazptr_rec* hprec) {
+ hprec_ = hprec;
+ DEBUG_PRINT(this << " " << hprec);
+}
+
+FOLLY_ALWAYS_INLINE hazptr_rec* hazptr_tc_entry::get() {
+ auto hprec = hprec_;
+ DEBUG_PRINT(this << " " << hprec);
+ return hprec;
+}
+
+inline void hazptr_tc_entry::evict() {
+ auto hprec = hprec_;
+ hprec->release();
+ DEBUG_PRINT(this << " " << hprec);
+}
+
+/** hazptr_tc */
+
+FOLLY_ALWAYS_INLINE hazptr_tc_entry& hazptr_tc::operator[](size_t i) {
+ DCHECK(i <= HAZPTR_TC_SIZE);
+ return entry_[i];
+}
+
+FOLLY_ALWAYS_INLINE hazptr_rec* hazptr_tc::get() {
+ if (LIKELY(count_ != 0)) {
+ auto hprec = entry_[--count_].get();
+ DEBUG_PRINT(this << " " << hprec);
+ return hprec;
+ }
+ DEBUG_PRINT(this << " nullptr");
+ return nullptr;
+}
+
+FOLLY_ALWAYS_INLINE bool hazptr_tc::put(hazptr_rec* hprec) {
+ if (LIKELY(count_ < HAZPTR_TC_SIZE)) {
+ entry_[count_++].fill(hprec);
+ DEBUG_PRINT(this << " " << count_ - 1);
+ return true;
+ }
+ return false;
+}
+
+FOLLY_ALWAYS_INLINE size_t hazptr_tc::count() {
+ return count_;
+}
+
+/** hazptr_tc free functions */
+
+FOLLY_ALWAYS_INLINE hazptr_tc* hazptr_tc_tls() {
+ DEBUG_PRINT(tls_state_);
+ if (LIKELY(tls_state_ == TLS_ALIVE)) {
+ DEBUG_PRINT(tls_state_);
+ return &tls_tc_data_;
+ } else if (tls_state_ == TLS_UNINITIALIZED) {
+ tls_life_odr_use();
+ return &tls_tc_data_;
+ }
+ return nullptr;
+}
+
+inline void hazptr_tc_init() {
+ DEBUG_PRINT("");
+ auto& tc = tls_tc_data_;
+ DEBUG_PRINT(&tc);
+ tc.count_ = 0;
+#ifndef NDEBUG
+ tc.local_ = false;
+#endif
+}
+
+inline void hazptr_tc_shutdown() {
+ auto& tc = tls_tc_data_;
+ DEBUG_PRINT(&tc);
+ for (size_t i = 0; i < tc.count_; ++i) {
+ tc.entry_[i].evict();
+ }
+}
+
+FOLLY_ALWAYS_INLINE hazptr_rec* hazptr_tc_try_get() {
+ DEBUG_PRINT(TLS_UNINITIALIZED << TLS_ALIVE << TLS_DESTROYED);
+ DEBUG_PRINT(tls_state_);
+ if (LIKELY(tls_state_ == TLS_ALIVE)) {
+ DEBUG_PRINT(tls_state_);
+ return tls_tc_data_.get();
+ } else if (tls_state_ == TLS_UNINITIALIZED) {
+ tls_life_odr_use();
+ return tls_tc_data_.get();
+ }
+ return nullptr;
+}
+
+FOLLY_ALWAYS_INLINE bool hazptr_tc_try_put(hazptr_rec* hprec) {
+ DEBUG_PRINT(tls_state_);
+ if (LIKELY(tls_state_ == TLS_ALIVE)) {
+ DEBUG_PRINT(tls_state_);
+ return tls_tc_data_.put(hprec);
+ }
+ return false;
+}
+
+/**
+ * hazptr_priv
+ */
+
+inline void hazptr_priv::push(hazptr_obj* obj) {
+ auto& domain = default_hazptr_domain();
+ obj->next_ = nullptr;
+ if (tail_) {
+ tail_->next_ = obj;
+ } else {
+ if (!active_) {
+ domain.objRetire(obj);
+ return;
+ }
+ head_ = obj;
+ }
+ tail_ = obj;
+ ++rcount_;
+ if (domain.reachedThreshold(rcount_)) {
+ pushAllToDomain();
+ }
+}
+
+inline void hazptr_priv::pushAllToDomain() {
+ auto& domain = default_hazptr_domain();
+ domain.pushRetired(head_, tail_, rcount_);
+ head_ = nullptr;
+ tail_ = nullptr;
+ rcount_ = 0;
+ domain.tryBulkReclaim();
+}
+
+inline void hazptr_priv_init() {
+ auto& priv = tls_priv_data_;
+ DEBUG_PRINT(&priv);
+ priv.head_ = nullptr;
+ priv.tail_ = nullptr;
+ priv.rcount_ = 0;
+ priv.active_ = true;
+}
+
+inline void hazptr_priv_shutdown() {
+ auto& priv = tls_priv_data_;
+ DEBUG_PRINT(&priv);
+ DCHECK(priv.active_);
+ priv.active_ = false;
+ if (priv.tail_) {
+ priv.pushAllToDomain();
+ }
+}
+
+inline bool hazptr_priv_try_retire(hazptr_obj* obj) {
+ DEBUG_PRINT(tls_state_);
+ if (tls_state_ == TLS_ALIVE) {
+ DEBUG_PRINT(tls_state_);
+ tls_priv_data_.push(obj);
+ return true;
+ } else if (tls_state_ == TLS_UNINITIALIZED) {
+ DEBUG_PRINT(tls_state_);
+ tls_life_odr_use();
+ tls_priv_data_.push(obj);
+ return true;
+ }
+ return false;
+}
+
+/** hazptr_tls_life */
+
+inline void tls_life_odr_use() {
+ DEBUG_PRINT(tls_state_);
+ CHECK(tls_state_ == TLS_UNINITIALIZED);
+ auto volatile tlsOdrUse = &tls_life_;
+ CHECK(tlsOdrUse != nullptr);
+ DEBUG_PRINT(tlsOdrUse);
+}
+
+inline hazptr_tls_life::hazptr_tls_life() {
+ DEBUG_PRINT(this);
+ CHECK(tls_state_ == TLS_UNINITIALIZED);
+ hazptr_tc_init();
+ hazptr_priv_init();
+ tls_state_ = TLS_ALIVE;
+}
+
+inline hazptr_tls_life::~hazptr_tls_life() {
+ DEBUG_PRINT(this);
+ CHECK(tls_state_ == TLS_ALIVE);
+ hazptr_tc_shutdown();
+ hazptr_priv_shutdown();
+ tls_state_ = TLS_DESTROYED;
+}
+