2 * Copyright 2014 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #ifndef FOLLY_DETAIL_THREADLOCALDETAIL_H_
18 #define FOLLY_DETAIL_THREADLOCALDETAIL_H_
27 #include <glog/logging.h>
29 #include <folly/Foreach.h>
30 #include <folly/Exception.h>
31 #include <folly/Malloc.h>
34 namespace threadlocal_detail {
37 * Base class for deleters.
41 virtual ~DeleterBase() { }
42 virtual void dispose(void* ptr, TLPDestructionMode mode) const = 0;
46 * Simple deleter class that calls delete on the passed-in pointer.
49 class SimpleDeleter : public DeleterBase {
51 virtual void dispose(void* ptr, TLPDestructionMode mode) const {
52 delete static_cast<Ptr>(ptr);
57 * Custom deleter that calls a given callable.
59 template <class Ptr, class Deleter>
60 class CustomDeleter : public DeleterBase {
62 explicit CustomDeleter(Deleter d) : deleter_(d) { }
63 virtual void dispose(void* ptr, TLPDestructionMode mode) const {
64 deleter_(static_cast<Ptr>(ptr), mode);
72 * POD wrapper around an element (a void*) and an associated deleter.
73 * This must be POD, as we memset() it to 0 and memcpy() it around.
75 struct ElementWrapper {
76 void dispose(TLPDestructionMode mode) {
78 DCHECK(deleter != nullptr);
79 deleter->dispose(ptr, mode);
97 DCHECK(ptr == nullptr);
98 DCHECK(deleter == nullptr);
101 // We leak a single object here but that is ok. If we used an
102 // object directly, there is a chance that the destructor will be
103 // called on that static object before any of the ElementWrappers
104 // are disposed and that isn't so nice.
105 static auto d = new SimpleDeleter<Ptr>();
112 template <class Ptr, class Deleter>
113 void set(Ptr p, Deleter d) {
114 DCHECK(ptr == nullptr);
115 DCHECK(deleter == nullptr);
118 deleter = new CustomDeleter<Ptr,Deleter>(d);
133 DeleterBase* deleter;
138 * Per-thread entry. Each thread using a StaticMeta object has one.
139 * This is written from the owning thread only (under the lock), read
140 * from the owning thread (no lock necessary), and read from other threads
144 ElementWrapper* elements;
145 size_t elementsCapacity;
150 // Held in a singleton to track our global instances.
151 // We have one of these per "Tag", by default one for the whole system
154 // Creating and destroying ThreadLocalPtr objects, as well as thread exit
155 // for threads that use ThreadLocalPtr objects collide on a lock inside
156 // StaticMeta; you can specify multiple Tag types to break that lock.
159 static StaticMeta<Tag>& instance() {
160 // Leak it on exit, there's only one per process and we don't have to
161 // worry about synchronization with exiting threads.
162 static bool constructed = (inst_ = new StaticMeta<Tag>());
163 (void)constructed; // suppress unused warning
168 std::vector<int> freeIds_;
170 pthread_key_t pthreadKey_;
173 void push_back(ThreadEntry* t) {
175 t->prev = head_.prev;
176 head_.prev->next = t;
180 void erase(ThreadEntry* t) {
181 t->next->prev = t->prev;
182 t->prev->next = t->next;
183 t->next = t->prev = t;
187 static FOLLY_TLS ThreadEntry threadEntry_;
189 static StaticMeta<Tag>* inst_;
191 StaticMeta() : nextId_(1) {
192 head_.next = head_.prev = &head_;
193 int ret = pthread_key_create(&pthreadKey_, &onThreadExit);
194 checkPosixError(ret, "pthread_key_create failed");
196 ret = pthread_atfork(/*prepare*/ &StaticMeta::preFork,
197 /*parent*/ &StaticMeta::onForkParent,
198 /*child*/ &StaticMeta::onForkChild);
199 checkPosixError(ret, "pthread_atfork failed");
202 LOG(FATAL) << "StaticMeta lives forever!";
205 static ThreadEntry* getThreadEntry() {
207 return &threadEntry_;
209 ThreadEntry* threadEntry =
210 static_cast<ThreadEntry*>(pthread_getspecific(inst_->pthreadKey_));
212 threadEntry = new ThreadEntry();
213 int ret = pthread_setspecific(inst_->pthreadKey_, threadEntry);
214 checkPosixError(ret, "pthread_setspecific failed");
220 static void preFork(void) {
221 instance().lock_.lock(); // Make sure it's created
224 static void onForkParent(void) {
225 inst_->lock_.unlock();
228 static void onForkChild(void) {
229 // only the current thread survives
230 inst_->head_.next = inst_->head_.prev = &inst_->head_;
231 ThreadEntry* threadEntry = getThreadEntry();
232 // If this thread was in the list before the fork, add it back.
233 if (threadEntry->elementsCapacity != 0) {
234 inst_->push_back(threadEntry);
236 inst_->lock_.unlock();
239 static void onThreadExit(void* ptr) {
240 auto & meta = instance();
242 ThreadEntry* threadEntry = getThreadEntry();
244 DCHECK_EQ(ptr, &meta);
245 DCHECK_GT(threadEntry->elementsCapacity, 0);
247 ThreadEntry* threadEntry = static_cast<ThreadEntry*>(ptr);
250 std::lock_guard<std::mutex> g(meta.lock_);
251 meta.erase(threadEntry);
252 // No need to hold the lock any longer; the ThreadEntry is private to this
253 // thread now that it's been removed from meta.
255 FOR_EACH_RANGE(i, 0, threadEntry->elementsCapacity) {
256 threadEntry->elements[i].dispose(TLPDestructionMode::THIS_THREAD);
258 free(threadEntry->elements);
259 threadEntry->elements = nullptr;
260 pthread_setspecific(meta.pthreadKey_, nullptr);
263 // Allocated in getThreadEntry(); free it
268 static int create() {
270 auto & meta = instance();
271 std::lock_guard<std::mutex> g(meta.lock_);
272 if (!meta.freeIds_.empty()) {
273 id = meta.freeIds_.back();
274 meta.freeIds_.pop_back();
281 static void destroy(size_t id) {
283 auto & meta = instance();
284 // Elements in other threads that use this id.
285 std::vector<ElementWrapper> elements;
287 std::lock_guard<std::mutex> g(meta.lock_);
288 for (ThreadEntry* e = meta.head_.next; e != &meta.head_; e = e->next) {
289 if (id < e->elementsCapacity && e->elements[id].ptr) {
290 elements.push_back(e->elements[id]);
293 * Writing another thread's ThreadEntry from here is fine;
294 * the only other potential reader is the owning thread --
295 * from onThreadExit (which grabs the lock, so is properly
296 * synchronized with us) or from get(), which also grabs
297 * the lock if it needs to resize the elements vector.
299 * We can't conflict with reads for a get(id), because
300 * it's illegal to call get on a thread local that's
303 e->elements[id].ptr = nullptr;
304 e->elements[id].deleter = nullptr;
305 e->elements[id].ownsDeleter = false;
308 meta.freeIds_.push_back(id);
310 // Delete elements outside the lock
311 FOR_EACH(it, elements) {
312 it->dispose(TLPDestructionMode::ALL_THREADS);
314 } catch (...) { // Just in case we get a lock error or something anyway...
315 LOG(WARNING) << "Destructor discarding an exception that was thrown.";
320 * Reserve enough space in the ThreadEntry::elements for the item
323 static void reserve(int id) {
324 auto& meta = instance();
325 ThreadEntry* threadEntry = getThreadEntry();
326 size_t prevCapacity = threadEntry->elementsCapacity;
327 // Growth factor < 2, see folly/docs/FBVector.md; + 5 to prevent
329 size_t newCapacity = static_cast<size_t>((id + 5) * 1.7);
330 assert(newCapacity > prevCapacity);
331 ElementWrapper* reallocated = nullptr;
333 // Need to grow. Note that we can't call realloc, as elements is
334 // still linked in meta, so another thread might access invalid memory
335 // after realloc succeeds. We'll copy by hand and update our ThreadEntry
337 if (usingJEMalloc()) {
338 bool success = false;
339 size_t newByteSize = newCapacity * sizeof(ElementWrapper);
340 size_t realByteSize = 0;
342 // Try to grow in place.
344 // Note that rallocm(ALLOCM_ZERO) will only zero newly allocated memory,
345 // even if a previous allocation allocated more than we requested.
346 // This is fine; we always use ALLOCM_ZERO with jemalloc and we
347 // always expand our allocation to the real size.
348 if (prevCapacity * sizeof(ElementWrapper) >=
349 jemallocMinInPlaceExpandable) {
350 success = (rallocm(reinterpret_cast<void**>(&threadEntry->elements),
354 ALLOCM_NO_MOVE | ALLOCM_ZERO) == ALLOCM_SUCCESS);
358 // In-place growth failed.
360 // Note that, unlike calloc,allocm(... ALLOCM_ZERO) zeros all
361 // allocated bytes (*realByteSize) and not just the requested
362 // bytes (newByteSize)
363 success = (allocm(reinterpret_cast<void**>(&reallocated),
366 ALLOCM_ZERO) == ALLOCM_SUCCESS);
370 // Expand to real size
371 assert(realByteSize / sizeof(ElementWrapper) >= newCapacity);
372 newCapacity = realByteSize / sizeof(ElementWrapper);
374 throw std::bad_alloc();
376 } else { // no jemalloc
377 // calloc() is simpler than malloc() followed by memset(), and
378 // potentially faster when dealing with a lot of memory, as it can get
379 // already-zeroed pages from the kernel.
380 reallocated = static_cast<ElementWrapper*>(
381 calloc(newCapacity, sizeof(ElementWrapper)));
383 throw std::bad_alloc();
387 // Success, update the entry
389 std::lock_guard<std::mutex> g(meta.lock_);
391 if (prevCapacity == 0) {
392 meta.push_back(threadEntry);
397 * Note: we need to hold the meta lock when copying data out of
398 * the old vector, because some other thread might be
399 * destructing a ThreadLocal and writing to the elements vector
402 memcpy(reallocated, threadEntry->elements,
403 sizeof(ElementWrapper) * prevCapacity);
405 swap(reallocated, threadEntry->elements);
407 threadEntry->elementsCapacity = newCapacity;
413 if (prevCapacity == 0) {
414 pthread_setspecific(meta.pthreadKey_, &meta);
419 static ElementWrapper& get(size_t id) {
420 ThreadEntry* threadEntry = getThreadEntry();
421 if (UNLIKELY(threadEntry->elementsCapacity <= id)) {
423 assert(threadEntry->elementsCapacity > id);
425 return threadEntry->elements[id];
431 FOLLY_TLS ThreadEntry StaticMeta<Tag>::threadEntry_ = {0};
433 template <class Tag> StaticMeta<Tag>* StaticMeta<Tag>::inst_ = nullptr;
435 } // namespace threadlocal_detail
438 #endif /* FOLLY_DETAIL_THREADLOCALDETAIL_H_ */