Split get_default() into two for deferred default construction and added forwarding...

[folly.git] / folly / detail / ThreadLocalDetail.h

diff --git a/folly/detail/ThreadLocalDetail.h b/folly/detail/ThreadLocalDetail.h
index 3d5a2aa83c34433b0174399bbeccf3011771739d..4a88e1bff26663d216912ed1ca4871d0713bfd85 100644 (file)
--- a/folly/detail/ThreadLocalDetail.h
+++ b/folly/detail/ThreadLocalDetail.h
@@ -33,6 +33,7 @@
 #include <folly/ScopeGuard.h>
 #include <folly/SharedMutex.h>
 #include <folly/container/Foreach.h>
+#include <folly/detail/AtFork.h>
 #include <folly/memory/Malloc.h>
 #include <folly/portability/PThread.h>
 
@@ -290,13 +291,7 @@ struct StaticMetaBase {
    */
   void reserve(EntryID* id);
 
-  ElementWrapper& get(EntryID* ent);
-
-  static void initAtFork();
-  static void registerAtFork(
-      folly::Function<void()> prepare,
-      folly::Function<void()> parent,
-      folly::Function<void()> child);
+  ElementWrapper& getElement(EntryID* ent);
 
   uint32_t nextId_;
   std::vector<uint32_t> freeIds_;
@@ -321,7 +316,8 @@ struct StaticMeta : StaticMetaBase {
       : StaticMetaBase(
             &StaticMeta::getThreadEntrySlow,
             std::is_same<AccessMode, AccessModeStrict>::value) {
-    registerAtFork(
+    detail::AtFork::registerHandler(
+        this,
         /*prepare*/ &StaticMeta::preFork,
         /*parent*/ &StaticMeta::onForkParent,
         /*child*/ &StaticMeta::onForkChild);
@@ -335,17 +331,37 @@ struct StaticMeta : StaticMetaBase {
     return *instance;
   }
 
-  ElementWrapper& get(EntryID* ent) {
-    ThreadEntry* threadEntry = getThreadEntry();
+  FOLLY_ALWAYS_INLINE static ElementWrapper& get(EntryID* ent) {
     uint32_t id = ent->getOrInvalid();
-    // if id is invalid, it is equal to uint32_t's max value.
-    // x <= max value is always true
+#ifdef FOLLY_TLD_USE_FOLLY_TLS
+    static FOLLY_TLS ThreadEntry* threadEntry{};
+    static FOLLY_TLS size_t capacity{};
+    // Eliminate as many branches and as much extra code as possible in the
+    // cached fast path, leaving only one branch here and one indirection below.
+    if (UNLIKELY(capacity <= id)) {
+      getSlowReserveAndCache(ent, id, threadEntry, capacity);
+    }
+#else
+    ThreadEntry* threadEntry{};
+    size_t capacity{};
+    getSlowReserveAndCache(ent, id, threadEntry, capacity);
+#endif
+    return threadEntry->elements[id];
+  }
+
+  static void getSlowReserveAndCache(
+      EntryID* ent,
+      uint32_t& id,
+      ThreadEntry*& threadEntry,
+      size_t& capacity) {
+    auto& inst = instance();
+    threadEntry = inst.threadEntry_();
     if (UNLIKELY(threadEntry->elementsCapacity <= id)) {
-      reserve(ent);
+      inst.reserve(ent);
       id = ent->getOrInvalid();
-      assert(threadEntry->elementsCapacity > id);
     }
-    return threadEntry->elements[id];
+    capacity = threadEntry->elementsCapacity;
+    assert(capacity > id);
   }
 
   static ThreadEntry* getThreadEntrySlow() {
@@ -367,18 +383,6 @@ struct StaticMeta : StaticMetaBase {
     return threadEntry;
   }
 
-  inline static ThreadEntry* getThreadEntry() {
-#ifdef FOLLY_TLD_USE_FOLLY_TLS
-    static FOLLY_TLS ThreadEntry* threadEntryCache{nullptr};
-    if (UNLIKELY(threadEntryCache == nullptr)) {
-      threadEntryCache = instance().threadEntry_();
-    }
-    return threadEntryCache;
-#else
-    return instance().threadEntry_();
-#endif
-  }
-
   static void preFork() {
     instance().lock_.lock();  // Make sure it's created
   }
@@ -390,7 +394,7 @@ struct StaticMeta : StaticMetaBase {
   static void onForkChild() {
     // only the current thread survives
     instance().head_.next = instance().head_.prev = &instance().head_;
-    ThreadEntry* threadEntry = getThreadEntry();
+    ThreadEntry* threadEntry = instance().threadEntry_();
     // If this thread was in the list before the fork, add it back.
     if (threadEntry->elementsCapacity != 0) {
       instance().push_back(threadEntry);
@@ -398,6 +402,5 @@ struct StaticMeta : StaticMetaBase {
     instance().lock_.unlock();
   }
 };
-
 } // namespace threadlocal_detail
 } // namespace folly