Use membarrier in TLRefCount
authorAndrii Grynenko <andrii@fb.com>
Tue, 19 Jul 2016 01:17:06 +0000 (18:17 -0700)
committerFacebook Github Bot 5 <facebook-github-bot-5-bot@fb.com>
Tue, 19 Jul 2016 01:23:30 +0000 (18:23 -0700)
Summary: membarrier guarantees that there's at most one update to thread-local counter, which collecting thread may not see.

Reviewed By: djwatson

Differential Revision: D3532952

fbshipit-source-id: 6106bfe87c70c5f864573a424662778e20423bbb

folly/Makefile.am
folly/Portability.h
folly/experimental/AsymmetricMemoryBarrier.cpp [new file with mode: 0644]
folly/experimental/AsymmetricMemoryBarrier.h [new file with mode: 0644]
folly/experimental/TLRefCount.h
folly/portability/SysMembarrier.cpp [new file with mode: 0644]
folly/portability/SysMembarrier.h [new file with mode: 0644]

index e71ec9a..1dd5e1f 100644 (file)
@@ -89,6 +89,7 @@ nobase_follyinclude_HEADERS = \
        ExceptionWrapper.h \
        Executor.h \
        EvictingCacheMap.h \
+       experimental/AsymmetricMemoryBarrier.h \
        experimental/AutoTimer.h \
        experimental/Bits.h \
        experimental/BitVectorCoding.h \
@@ -273,6 +274,7 @@ nobase_follyinclude_HEADERS = \
        portability/String.h \
        portability/Syslog.h \
        portability/SysFile.h \
+       portability/SysMembarrier.h \
        portability/SysMman.h \
        portability/SysResource.h \
        portability/SysStat.h \
@@ -428,6 +430,7 @@ libfolly_la_SOURCES = \
        portability/Stdlib.cpp \
        portability/String.cpp \
        portability/SysFile.cpp \
+       portability/SysMembarrier.cpp \
        portability/SysMman.cpp \
        portability/SysResource.cpp \
        portability/SysStat.cpp \
@@ -450,6 +453,7 @@ libfolly_la_SOURCES = \
        TimeoutQueue.cpp \
        Uri.cpp \
        Version.cpp \
+       experimental/AsymmetricMemoryBarrier.cpp \
        experimental/bser/Dump.cpp \
        experimental/bser/Load.cpp \
        experimental/DynamicParser.cpp \
index 2d3ec41..5776a00 100644 (file)
@@ -354,3 +354,12 @@ using namespace FOLLY_GFLAGS_NAMESPACE;
 // we will take the next one.
 #define FOLLY_STATIC_CTOR_PRIORITY_MAX __attribute__((__init_priority__(102)))
 #endif
+
+namespace folly {
+
+#ifdef __linux__
+constexpr auto kIsLinux = true;
+#else
+constexpr auto kIsLinux = false;
+#endif
+}
diff --git a/folly/experimental/AsymmetricMemoryBarrier.cpp b/folly/experimental/AsymmetricMemoryBarrier.cpp
new file mode 100644 (file)
index 0000000..31c34cb
--- /dev/null
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2016 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "AsymmetricMemoryBarrier.h"
+
+#include <folly/Exception.h>
+#include <folly/Indestructible.h>
+#include <folly/portability/SysMembarrier.h>
+#include <folly/portability/SysMman.h>
+#include <mutex>
+
+namespace folly {
+
+namespace {
+
+struct DummyPageCreator {
+  DummyPageCreator() {
+    get();
+  }
+
+  static void* get() {
+    static auto ptr =
+        kIsLinux && !detail::sysMembarrierAvailable() ? create() : nullptr;
+    return ptr;
+  }
+
+ private:
+  static void* create() {
+    auto ptr = mmap(nullptr, 1, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    checkUnixError(reinterpret_cast<uintptr_t>(ptr), "mmap");
+
+    // Lock the memory so it can't get paged out. If it gets paged out, changing
+    // its protection won't accomplish anything.
+    auto r = mlock(ptr, 1);
+    checkUnixError(r, "mlock");
+
+    return ptr;
+  }
+};
+
+// Make sure dummy page is always initialized before shutdown
+DummyPageCreator dummyPageCreator;
+
+void mprotectMembarrier() {
+  auto dummyPage = dummyPageCreator.get();
+
+  // This function is required to be safe to call on shutdown,
+  // so we must leak the mutex.
+  static Indestructible<std::mutex> mprotectMutex;
+  std::lock_guard<std::mutex> lg(*mprotectMutex);
+
+  int r = 0;
+  r = mprotect(dummyPage, 1, PROT_READ | PROT_WRITE);
+  checkUnixError(r, "mprotect");
+
+  r = mprotect(dummyPage, 1, PROT_READ);
+  checkUnixError(r, "mprotect");
+}
+}
+
+void asymmetricHeavyBarrier() {
+  if (kIsLinux) {
+    static const bool useSysMembarrier = detail::sysMembarrierAvailable();
+    if (useSysMembarrier) {
+      auto r = detail::sysMembarrier();
+      checkUnixError(r, "membarrier");
+    } else {
+      mprotectMembarrier();
+    }
+  } else {
+    std::atomic_thread_fence(std::memory_order_seq_cst);
+  }
+}
+}
diff --git a/folly/experimental/AsymmetricMemoryBarrier.h b/folly/experimental/AsymmetricMemoryBarrier.h
new file mode 100644 (file)
index 0000000..14573cc
--- /dev/null
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2016 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <atomic>
+
+#include <folly/portability/Asm.h>
+
+namespace folly {
+
+FOLLY_ALWAYS_INLINE void asymmetricLightBarrier() {
+  if (kIsLinux) {
+    asm_volatile_memory();
+  } else {
+    std::atomic_thread_fence(std::memory_order_seq_cst);
+  }
+}
+
+void asymmetricHeavyBarrier();
+}
index 9667f4c..78054ed 100644 (file)
@@ -16,6 +16,7 @@
 #pragma once
 
 #include <folly/ThreadLocal.h>
+#include <folly/experimental/AsymmetricMemoryBarrier.h>
 
 namespace folly {
 
@@ -84,6 +85,8 @@ class TLRefCount {
 
     state_ = State::GLOBAL_TRANSITION;
 
+    asymmetricHeavyBarrier();
+
     std::weak_ptr<void> collectGuardWeak = collectGuard_;
 
     // Make sure we can't create new LocalRefCounts
@@ -147,7 +150,14 @@ class TLRefCount {
         return false;
       }
 
-      auto count = count_ += delta;
+      // This is equivalent to atomic fetch_add. We know that this operation
+      // is always performed from a single thread. asymmetricLightBarrier()
+      // makes things faster than atomic fetch_add on platforms with native
+      // support.
+      auto count = count_.load(std::memory_order_relaxed) + delta;
+      count_.store(count, std::memory_order_relaxed);
+
+      asymmetricLightBarrier();
 
       if (UNLIKELY(refCount_.state_.load() != State::LOCAL)) {
         std::lock_guard<std::mutex> lg(collectMutex_);
diff --git a/folly/portability/SysMembarrier.cpp b/folly/portability/SysMembarrier.cpp
new file mode 100644 (file)
index 0000000..923bece
--- /dev/null
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2016 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "SysMembarrier.h"
+
+#include <mutex>
+
+#include <folly/Portability.h>
+#include <folly/portability/Unistd.h>
+
+#if !defined(__NR_membarrier) && defined(FOLLY_X64)
+#define __NR_membarrier 324
+#define MEMBARRIER_CMD_QUERY 0
+#define MEMBARRIER_CMD_SHARED 1
+#endif
+
+namespace folly {
+namespace detail {
+
+bool sysMembarrierAvailable() {
+  if (!kIsLinux) {
+    return false;
+  }
+
+#ifdef __NR_membarrier
+  auto r = syscall(__NR_membarrier, MEMBARRIER_CMD_QUERY, /* flags = */ 0);
+  if (r == -1) {
+    return false;
+  }
+
+  return r & MEMBARRIER_CMD_SHARED;
+#else
+  return false;
+#endif
+}
+
+int sysMembarrier() {
+#ifdef __NR_membarrier
+  return syscall(__NR_membarrier, MEMBARRIER_CMD_SHARED, /* flags = */ 0);
+#else
+  return -1;
+#endif
+}
+}
+}
diff --git a/folly/portability/SysMembarrier.h b/folly/portability/SysMembarrier.h
new file mode 100644 (file)
index 0000000..01284b6
--- /dev/null
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2016 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+namespace folly {
+namespace detail {
+
+int sysMembarrier();
+bool sysMembarrierAvailable();
+}
+}