Let Futex import base-class ctors

[folly.git] / folly / SharedMutex.h

diff --git a/folly/SharedMutex.h b/folly/SharedMutex.h
index 48041c118896765ffe0ff2383488fe2a45b8a061..976c6e09f80f9918aa1f599167aea16a00ab6f52 100644 (file)
--- a/folly/SharedMutex.h
+++ b/folly/SharedMutex.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2016 Facebook, Inc.
+ * Copyright 2017 Facebook, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,11 +19,13 @@
 #pragma once
 
 #include <stdint.h>
+
 #include <atomic>
 #include <thread>
 #include <type_traits>
+
 #include <folly/Likely.h>
-#include <folly/detail/CacheLocality.h>
+#include <folly/concurrency/CacheLocality.h>
 #include <folly/detail/Futex.h>
 #include <folly/portability/Asm.h>
 #include <folly/portability/SysResource.h>
@@ -234,10 +236,11 @@ struct SharedMutexToken {
   uint16_t slot_;
 };
 
-template <bool ReaderPriority,
-          typename Tag_ = void,
-          template <typename> class Atom = std::atomic,
-          bool BlockImmediately = false>
+template <
+    bool ReaderPriority,
+    typename Tag_ = void,
+    template <typename> class Atom = std::atomic,
+    bool BlockImmediately = false>
 class SharedMutexImpl {
  public:
   static constexpr bool kReaderPriority = ReaderPriority;
@@ -249,7 +252,7 @@ class SharedMutexImpl {
   class UpgradeHolder;
   class WriteHolder;
 
-  constexpr SharedMutexImpl() : state_(0) {}
+  constexpr SharedMutexImpl() noexcept : state_(0) {}
 
   SharedMutexImpl(const SharedMutexImpl&) = delete;
   SharedMutexImpl(SharedMutexImpl&&) = delete;
@@ -560,7 +563,7 @@ class SharedMutexImpl {
   };
 
   // 32 bits of state
-  Futex state_;
+  Futex state_{};
 
   // S count needs to be on the end, because we explicitly allow it to
   // underflow.  This can occur while we are in the middle of applying
@@ -735,9 +738,8 @@ class SharedMutexImpl {
   typedef Atom<uintptr_t> DeferredReaderSlot;
 
  private:
-  FOLLY_ALIGN_TO_AVOID_FALSE_SHARING static DeferredReaderSlot deferredReaders
-      [kMaxDeferredReaders *
-       kDeferredSeparationFactor];
+  alignas(hardware_destructive_interference_size) static DeferredReaderSlot
+      deferredReaders[kMaxDeferredReaders * kDeferredSeparationFactor];
 
   // Performs an exclusive lock, waiting for state_ & waitMask to be
   // zero first
@@ -765,7 +767,7 @@ class SharedMutexImpl {
       }
 
       uint32_t after = (state & kMayDefer) == 0 ? 0 : kPrevDefer;
-      if (!ReaderPriority || (state & (kMayDefer | kHasS)) == 0) {
+      if (!kReaderPriority || (state & (kMayDefer | kHasS)) == 0) {
         // Block readers immediately, either because we are in write
         // priority mode or because we can acquire the lock in one
         // step.  Note that if state has kHasU, then we are doing an
@@ -806,7 +808,7 @@ class SharedMutexImpl {
             return false;
           }
 
-          if (ReaderPriority && (state & kHasE) == 0) {
+          if (kReaderPriority && (state & kHasE) == 0) {
             assert((state & kBegunE) != 0);
             if (!state_.compare_exchange_strong(state,
                                                 (state & ~kBegunE) | kHasE)) {
@@ -847,6 +849,7 @@ class SharedMutexImpl {
                             WaitContext& ctx) {
 #ifdef RUSAGE_THREAD
     struct rusage usage;
+    std::memset(&usage, 0, sizeof(usage));
     long before = -1;
 #endif
     for (uint32_t yieldCount = 0; yieldCount < kMaxSoftYieldCount;
@@ -989,7 +992,7 @@ class SharedMutexImpl {
           return;
         }
       }
-      asm_pause();
+      asm_volatile_pause();
       if (UNLIKELY(++spinCount >= kMaxSpinCount)) {
         applyDeferredReaders(state, ctx, slot);
         return;
@@ -1002,6 +1005,7 @@ class SharedMutexImpl {
 
 #ifdef RUSAGE_THREAD
     struct rusage usage;
+    std::memset(&usage, 0, sizeof(usage));
     long before = -1;
 #endif
     for (uint32_t yieldCount = 0; yieldCount < kMaxSoftYieldCount;
@@ -1133,10 +1137,15 @@ class SharedMutexImpl {
 
  public:
   class ReadHolder {
-   public:
     ReadHolder() : lock_(nullptr) {}
 
-    explicit ReadHolder(const SharedMutexImpl* lock) : ReadHolder(*lock) {}
+   public:
+    explicit ReadHolder(const SharedMutexImpl* lock)
+        : lock_(const_cast<SharedMutexImpl*>(lock)) {
+      if (lock_) {
+        lock_->lock_shared(token_);
+      }
+    }
 
     explicit ReadHolder(const SharedMutexImpl& lock)
         : lock_(const_cast<SharedMutexImpl*>(&lock)) {
@@ -1190,10 +1199,14 @@ class SharedMutexImpl {
   };
 
   class UpgradeHolder {
-   public:
     UpgradeHolder() : lock_(nullptr) {}
 
-    explicit UpgradeHolder(SharedMutexImpl* lock) : UpgradeHolder(*lock) {}
+   public:
+    explicit UpgradeHolder(SharedMutexImpl* lock) : lock_(lock) {
+      if (lock_) {
+        lock_->lock_upgrade();
+      }
+    }
 
     explicit UpgradeHolder(SharedMutexImpl& lock) : lock_(&lock) {
       lock_->lock_upgrade();
@@ -1236,10 +1249,14 @@ class SharedMutexImpl {
   };
 
   class WriteHolder {
-   public:
     WriteHolder() : lock_(nullptr) {}
 
-    explicit WriteHolder(SharedMutexImpl* lock) : WriteHolder(*lock) {}
+   public:
+    explicit WriteHolder(SharedMutexImpl* lock) : lock_(lock) {
+      if (lock_) {
+        lock_->lock();
+      }
+    }
 
     explicit WriteHolder(SharedMutexImpl& lock) : lock_(&lock) {
       lock_->lock();
@@ -1332,11 +1349,11 @@ template <
     typename Tag_,
     template <typename> class Atom,
     bool BlockImmediately>
-typename SharedMutexImpl<ReaderPriority, Tag_, Atom, BlockImmediately>::
-    DeferredReaderSlot
-        SharedMutexImpl<ReaderPriority, Tag_, Atom, BlockImmediately>::
-            deferredReaders[kMaxDeferredReaders * kDeferredSeparationFactor] =
-                {};
+alignas(hardware_destructive_interference_size)
+    typename SharedMutexImpl<ReaderPriority, Tag_, Atom, BlockImmediately>::
+        DeferredReaderSlot
+    SharedMutexImpl<ReaderPriority, Tag_, Atom, BlockImmediately>::
+        deferredReaders[kMaxDeferredReaders * kDeferredSeparationFactor] = {};
 
 template <
     bool ReaderPriority,
@@ -1404,8 +1421,7 @@ bool SharedMutexImpl<ReaderPriority, Tag_, Atom, BlockImmediately>::
         // starting point for our empty-slot search, can change after
         // calling waitForZeroBits
         uint32_t bestSlot =
-            (uint32_t)folly::detail::AccessSpreader<Atom>::current(
-                kMaxDeferredReaders);
+            (uint32_t)folly::AccessSpreader<Atom>::current(kMaxDeferredReaders);
 
         // deferred readers are already enabled, or it is time to
         // enable them if we can find a slot