*/
#pragma once
-#include <folly/Baton.h>
#include <folly/ThreadLocal.h>
+#include <folly/experimental/AsymmetricMemoryBarrier.h>
namespace folly {
public:
using Int = int64_t;
- TLRefCount() :
- localCount_([&]() {
- return new LocalRefCount(*this);
- }),
- collectGuard_(&collectBaton_, [](void* p) {
- auto baton = reinterpret_cast<folly::Baton<>*>(p);
- baton->post();
- }) {
- }
+ TLRefCount()
+ : localCount_([&]() { return new LocalRefCount(*this); }),
+ collectGuard_(this, [](void*) {}) {}
~TLRefCount() noexcept {
assert(globalCount_.load() == 0);
}
void useGlobal() noexcept {
- std::lock_guard<std::mutex> lg(globalMutex_);
+ std::array<TLRefCount*, 1> ptrs{{this}};
+ useGlobal(ptrs);
+ }
- state_ = State::GLOBAL_TRANSITION;
+ template <typename Container>
+ static void useGlobal(const Container& refCountPtrs) {
+ std::vector<std::unique_lock<std::mutex>> lgs_;
+ for (auto refCountPtr : refCountPtrs) {
+ lgs_.emplace_back(refCountPtr->globalMutex_);
- auto accessor = localCount_.accessAllThreads();
- for (auto& count : accessor) {
- count.collect();
+ refCountPtr->state_ = State::GLOBAL_TRANSITION;
}
- collectGuard_.reset();
- collectBaton_.wait();
+ asymmetricHeavyBarrier();
+
+ for (auto refCountPtr : refCountPtrs) {
+ std::weak_ptr<void> collectGuardWeak = refCountPtr->collectGuard_;
- state_ = State::GLOBAL;
+ // Make sure we can't create new LocalRefCounts
+ refCountPtr->collectGuard_.reset();
+
+ while (!collectGuardWeak.expired()) {
+ auto accessor = refCountPtr->localCount_.accessAllThreads();
+ for (auto& count : accessor) {
+ count.collect();
+ }
+ }
+
+ refCountPtr->state_ = State::GLOBAL;
+ }
}
private:
return;
}
- collectCount_ = count_;
+ collectCount_ = count_.load();
refCount_.globalCount_.fetch_add(collectCount_);
collectGuard_.reset();
}
return false;
}
- auto count = count_ += delta;
+ // This is equivalent to atomic fetch_add. We know that this operation
+ // is always performed from a single thread. asymmetricLightBarrier()
+ // makes things faster than atomic fetch_add on platforms with native
+ // support.
+ auto count = count_.load(std::memory_order_relaxed) + delta;
+ count_.store(count, std::memory_order_relaxed);
+
+ asymmetricLightBarrier();
if (UNLIKELY(refCount_.state_.load() != State::LOCAL)) {
std::lock_guard<std::mutex> lg(collectMutex_);
return true;
}
- Int count_{0};
+ AtomicInt count_{0};
TLRefCount& refCount_;
std::mutex collectMutex_;
folly::ThreadLocal<LocalRefCount, TLRefCount> localCount_;
std::atomic<int64_t> globalCount_{1};
std::mutex globalMutex_;
- folly::Baton<> collectBaton_;
std::shared_ptr<void> collectGuard_;
};