/*
- * Copyright 2013 Facebook, Inc.
+ * Copyright 2015 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
#include <glog/logging.h>
-#include "folly/Foreach.h"
-#include "folly/Exception.h"
-#include "folly/Malloc.h"
+#include <folly/Foreach.h>
+#include <folly/Exception.h>
+#include <folly/Malloc.h>
-// TODO(tudorb): Remove this declaration after Malloc.h is pushed to
-// third-party.
-extern "C" int allocm(void**, size_t*, size_t, int)
-__attribute__((weak));
+// In general, emutls cleanup is not guaranteed to play nice with the way
+// StaticMeta mixes direct pthread calls and the use of __thread. This has
+// caused problems on multiple platforms so don't use __thread there.
+//
+// XXX: Ideally we would instead determine if emutls is in use at runtime as it
+// is possible to configure glibc on Linux to use emutls regardless.
+#if !__APPLE__ && !__ANDROID__
+#define FOLLY_TLD_USE_FOLLY_TLS 1
+#else
+#undef FOLLY_TLD_USE_FOLLY_TLS
+#endif
namespace folly {
namespace threadlocal_detail {
* This must be POD, as we memset() it to 0 and memcpy() it around.
*/
struct ElementWrapper {
- void dispose(TLPDestructionMode mode) {
- if (ptr != NULL) {
- DCHECK(deleter != NULL);
- deleter->dispose(ptr, mode);
- if (ownsDeleter) {
- delete deleter;
- }
- ptr = NULL;
- deleter = NULL;
- ownsDeleter = false;
+ bool dispose(TLPDestructionMode mode) {
+ if (ptr == nullptr) {
+ return false;
}
+
+ DCHECK(deleter != nullptr);
+ deleter->dispose(ptr, mode);
+ cleanup();
+ return true;
+ }
+
+ void* release() {
+ auto retPtr = ptr;
+
+ if (ptr != nullptr) {
+ cleanup();
+ }
+
+ return retPtr;
}
template <class Ptr>
void set(Ptr p) {
- DCHECK(ptr == NULL);
- DCHECK(deleter == NULL);
+ DCHECK(ptr == nullptr);
+ DCHECK(deleter == nullptr);
if (p) {
// We leak a single object here but that is ok. If we used an
template <class Ptr, class Deleter>
void set(Ptr p, Deleter d) {
- DCHECK(ptr == NULL);
- DCHECK(deleter == NULL);
+ DCHECK(ptr == nullptr);
+ DCHECK(deleter == nullptr);
if (p) {
ptr = p;
deleter = new CustomDeleter<Ptr,Deleter>(d);
}
}
+ void cleanup() {
+ if (ownsDeleter) {
+ delete deleter;
+ }
+ ptr = nullptr;
+ deleter = nullptr;
+ ownsDeleter = false;
+ }
+
void* ptr;
DeleterBase* deleter;
bool ownsDeleter;
return *inst_;
}
- int nextId_;
- std::vector<int> freeIds_;
+ uint32_t nextId_;
+ std::vector<uint32_t> freeIds_;
std::mutex lock_;
pthread_key_t pthreadKey_;
ThreadEntry head_;
t->next = t->prev = t;
}
- static __thread ThreadEntry threadEntry_;
+#ifdef FOLLY_TLD_USE_FOLLY_TLS
+ static FOLLY_TLS ThreadEntry threadEntry_;
+#endif
static StaticMeta<Tag>* inst_;
StaticMeta() : nextId_(1) {
int ret = pthread_key_create(&pthreadKey_, &onThreadExit);
checkPosixError(ret, "pthread_key_create failed");
+#if FOLLY_HAVE_PTHREAD_ATFORK
ret = pthread_atfork(/*prepare*/ &StaticMeta::preFork,
/*parent*/ &StaticMeta::onForkParent,
/*child*/ &StaticMeta::onForkChild);
checkPosixError(ret, "pthread_atfork failed");
+#elif !__ANDROID__
+ // pthread_atfork is not part of the Android NDK at least as of n9d. If
+ // something is trying to call native fork() directly at all with Android's
+ // process management model, this is probably the least of the problems.
+ //
+ // But otherwise, this is a problem.
+ #warning pthread_atfork unavailable
+#endif
}
~StaticMeta() {
LOG(FATAL) << "StaticMeta lives forever!";
}
+ static ThreadEntry* getThreadEntry() {
+#ifdef FOLLY_TLD_USE_FOLLY_TLS
+ return &threadEntry_;
+#else
+ ThreadEntry* threadEntry =
+ static_cast<ThreadEntry*>(pthread_getspecific(inst_->pthreadKey_));
+ if (!threadEntry) {
+ threadEntry = new ThreadEntry();
+ int ret = pthread_setspecific(inst_->pthreadKey_, threadEntry);
+ checkPosixError(ret, "pthread_setspecific failed");
+ }
+ return threadEntry;
+#endif
+ }
+
static void preFork(void) {
instance().lock_.lock(); // Make sure it's created
}
}
static void onForkChild(void) {
+ // only the current thread survives
inst_->head_.next = inst_->head_.prev = &inst_->head_;
- inst_->push_back(&threadEntry_); // only the current thread survives
+ ThreadEntry* threadEntry = getThreadEntry();
+ // If this thread was in the list before the fork, add it back.
+ if (threadEntry->elementsCapacity != 0) {
+ inst_->push_back(threadEntry);
+ }
inst_->lock_.unlock();
}
static void onThreadExit(void* ptr) {
- auto & meta = instance();
+ auto& meta = instance();
+#ifdef FOLLY_TLD_USE_FOLLY_TLS
+ ThreadEntry* threadEntry = getThreadEntry();
+
DCHECK_EQ(ptr, &meta);
- // We wouldn't call pthread_setspecific unless we actually called get()
- DCHECK_NE(threadEntry_.elementsCapacity, 0);
+ DCHECK_GT(threadEntry->elementsCapacity, 0);
+#else
+ ThreadEntry* threadEntry = static_cast<ThreadEntry*>(ptr);
+#endif
{
std::lock_guard<std::mutex> g(meta.lock_);
- meta.erase(&threadEntry_);
- // No need to hold the lock any longer; threadEntry_ is private to this
+ meta.erase(threadEntry);
+ // No need to hold the lock any longer; the ThreadEntry is private to this
// thread now that it's been removed from meta.
}
- FOR_EACH_RANGE(i, 0, threadEntry_.elementsCapacity) {
- threadEntry_.elements[i].dispose(TLPDestructionMode::THIS_THREAD);
+ // NOTE: User-provided deleter / object dtor itself may be using ThreadLocal
+ // with the same Tag, so dispose() calls below may (re)create some of the
+ // elements or even increase elementsCapacity, thus multiple cleanup rounds
+ // may be required.
+ for (bool shouldRun = true; shouldRun; ) {
+ shouldRun = false;
+ FOR_EACH_RANGE(i, 0, threadEntry->elementsCapacity) {
+ if (threadEntry->elements[i].dispose(TLPDestructionMode::THIS_THREAD)) {
+ shouldRun = true;
+ }
+ }
}
- free(threadEntry_.elements);
- threadEntry_.elements = NULL;
- pthread_setspecific(meta.pthreadKey_, NULL);
+ free(threadEntry->elements);
+ threadEntry->elements = nullptr;
+ pthread_setspecific(meta.pthreadKey_, nullptr);
+
+#ifndef FOLLY_TLD_USE_FOLLY_TLS
+ // Allocated in getThreadEntry() when not using folly TLS; free it
+ delete threadEntry;
+#endif
}
- static int create() {
- int id;
+ static uint32_t create() {
+ uint32_t id;
auto & meta = instance();
std::lock_guard<std::mutex> g(meta.lock_);
if (!meta.freeIds_.empty()) {
return id;
}
- static void destroy(int id) {
+ static void destroy(uint32_t id) {
try {
auto & meta = instance();
// Elements in other threads that use this id.
}
/**
- * Reserve enough space in the threadEntry_.elements for the item
+ * Reserve enough space in the ThreadEntry::elements for the item
* @id to fit in.
*/
- static void reserve(int id) {
- size_t prevCapacity = threadEntry_.elementsCapacity;
+ static void reserve(uint32_t id) {
+ auto& meta = instance();
+ ThreadEntry* threadEntry = getThreadEntry();
+ size_t prevCapacity = threadEntry->elementsCapacity;
// Growth factor < 2, see folly/docs/FBVector.md; + 5 to prevent
// very slow start.
size_t newCapacity = static_cast<size_t>((id + 5) * 1.7);
assert(newCapacity > prevCapacity);
- auto& meta = instance();
ElementWrapper* reallocated = nullptr;
// Need to grow. Note that we can't call realloc, as elements is
// still linked in meta, so another thread might access invalid memory
- // after realloc succeeds. We'll copy by hand and update threadEntry_
+ // after realloc succeeds. We'll copy by hand and update our ThreadEntry
// under the lock.
if (usingJEMalloc()) {
bool success = false;
- size_t newByteSize = newCapacity * sizeof(ElementWrapper);
- size_t realByteSize = 0;
+ size_t newByteSize = nallocx(newCapacity * sizeof(ElementWrapper), 0);
// Try to grow in place.
//
- // Note that rallocm(ALLOCM_ZERO) will only zero newly allocated memory,
+ // Note that xallocx(MALLOCX_ZERO) will only zero newly allocated memory,
// even if a previous allocation allocated more than we requested.
- // This is fine; we always use ALLOCM_ZERO with jemalloc and we
+ // This is fine; we always use MALLOCX_ZERO with jemalloc and we
// always expand our allocation to the real size.
if (prevCapacity * sizeof(ElementWrapper) >=
jemallocMinInPlaceExpandable) {
- success = (rallocm(reinterpret_cast<void**>(&threadEntry_.elements),
- &realByteSize,
- newByteSize,
- 0,
- ALLOCM_NO_MOVE | ALLOCM_ZERO) == ALLOCM_SUCCESS);
-
+ success = (xallocx(threadEntry->elements, newByteSize, 0, MALLOCX_ZERO)
+ == newByteSize);
}
// In-place growth failed.
if (!success) {
- // Note that, unlike calloc,allocm(... ALLOCM_ZERO) zeros all
- // allocated bytes (*realByteSize) and not just the requested
- // bytes (newByteSize)
- success = (allocm(reinterpret_cast<void**>(&reallocated),
- &realByteSize,
- newByteSize,
- ALLOCM_ZERO) == ALLOCM_SUCCESS);
+ success = ((reallocated = static_cast<ElementWrapper*>(
+ mallocx(newByteSize, MALLOCX_ZERO))) != nullptr);
}
if (success) {
// Expand to real size
- assert(realByteSize / sizeof(ElementWrapper) >= newCapacity);
- newCapacity = realByteSize / sizeof(ElementWrapper);
+ assert(newByteSize / sizeof(ElementWrapper) >= newCapacity);
+ newCapacity = newByteSize / sizeof(ElementWrapper);
} else {
throw std::bad_alloc();
}
std::lock_guard<std::mutex> g(meta.lock_);
if (prevCapacity == 0) {
- meta.push_back(&threadEntry_);
+ meta.push_back(threadEntry);
}
if (reallocated) {
* destructing a ThreadLocal and writing to the elements vector
* of this thread.
*/
- memcpy(reallocated, threadEntry_.elements,
+ memcpy(reallocated, threadEntry->elements,
sizeof(ElementWrapper) * prevCapacity);
using std::swap;
- swap(reallocated, threadEntry_.elements);
+ swap(reallocated, threadEntry->elements);
}
- threadEntry_.elementsCapacity = newCapacity;
+ threadEntry->elementsCapacity = newCapacity;
}
free(reallocated);
+#ifdef FOLLY_TLD_USE_FOLLY_TLS
if (prevCapacity == 0) {
pthread_setspecific(meta.pthreadKey_, &meta);
}
+#endif
}
- static ElementWrapper& get(int id) {
- if (UNLIKELY(threadEntry_.elementsCapacity <= id)) {
+ static ElementWrapper& get(uint32_t id) {
+ ThreadEntry* threadEntry = getThreadEntry();
+ if (UNLIKELY(threadEntry->elementsCapacity <= id)) {
reserve(id);
- assert(threadEntry_.elementsCapacity > id);
+ assert(threadEntry->elementsCapacity > id);
}
- return threadEntry_.elements[id];
+ return threadEntry->elements[id];
}
};
-template <class Tag> __thread ThreadEntry StaticMeta<Tag>::threadEntry_ = {0};
+#ifdef FOLLY_TLD_USE_FOLLY_TLS
+template <class Tag>
+FOLLY_TLS ThreadEntry StaticMeta<Tag>::threadEntry_{nullptr, 0,
+ nullptr, nullptr};
+#endif
template <class Tag> StaticMeta<Tag>* StaticMeta<Tag>::inst_ = nullptr;
} // namespace threadlocal_detail
} // namespace folly
#endif /* FOLLY_DETAIL_THREADLOCALDETAIL_H_ */
-