/*
- * Copyright 2013 Facebook, Inc.
+ * Copyright 2013-present Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
#pragma once
#include <atomic>
+#include <cassert>
#include <chrono>
#include <limits>
-#include <assert.h>
-#include <errno.h>
-#include <linux/futex.h>
-#include <sys/syscall.h>
-#include <unistd.h>
+#include <type_traits>
+
#include <boost/noncopyable.hpp>
-using std::chrono::steady_clock;
-using std::chrono::system_clock;
-using std::chrono::time_point;
+#include <folly/portability/Unistd.h>
namespace folly { namespace detail {
VALUE_CHANGED, /* Futex value didn't match expected */
AWOKEN, /* futex wait matched with a futex wake */
INTERRUPTED, /* Spurious wake-up or signal caused futex wait failure */
- TIMEDOUT
+ TIMEDOUT,
};
-/* Converts return value and errno from a futex syscall to a FutexResult */
-FutexResult futexErrnoToFutexResult(int returnVal, int futexErrno);
-
/**
* Futex is an atomic 32 bit unsigned integer that provides access to the
* futex() syscall on that value. It is templated in such a way that it
template <template <typename> class Atom = std::atomic>
struct Futex : Atom<uint32_t>, boost::noncopyable {
- explicit Futex(uint32_t init = 0) : Atom<uint32_t>(init) {}
+ explicit constexpr Futex(uint32_t init = 0) : Atom<uint32_t>(init) {}
/** Puts the thread to sleep if this->load() == expected. Returns true when
* it is returning because it has consumed a wake() event, false for any
* other return (signal, this->load() != expected, or spurious wakeup). */
- bool futexWait(uint32_t expected, uint32_t waitMask = -1);
+ FutexResult futexWait(uint32_t expected, uint32_t waitMask = -1) {
+ auto rv = futexWaitImpl(expected, nullptr, nullptr, waitMask);
+ assert(rv != FutexResult::TIMEDOUT);
+ return rv;
+ }
- /** Similar to futexWait but also accepts a timeout that gives the time until
- * when the call can block (time is the absolute time i.e time since epoch).
- * Allowed clock types: std::chrono::system_clock, std::chrono::steady_clock.
- * Returns one of FutexResult values.
+ /** Similar to futexWait but also accepts a deadline until when the wait call
+ * may block.
*
+ * Optimal clock types: std::chrono::system_clock, std::chrono::steady_clock.
* NOTE: On some systems steady_clock is just an alias for system_clock,
- * and is not actually steady.*/
+ * and is not actually steady.
+ *
+ * For any other clock type, now() will be invoked twice. */
template <class Clock, class Duration = typename Clock::duration>
- FutexResult futexWaitUntil(uint32_t expected,
- const time_point<Clock, Duration>& absTime,
- uint32_t waitMask = -1);
+ FutexResult futexWaitUntil(
+ uint32_t expected,
+ std::chrono::time_point<Clock, Duration> const& deadline,
+ uint32_t waitMask = -1) {
+ using Target = typename std::conditional<
+ Clock::is_steady,
+ std::chrono::steady_clock,
+ std::chrono::system_clock>::type;
+ auto const converted = time_point_conv<Target>(deadline);
+ return futexWaitImpl(expected, converted, waitMask);
+ }
- /** Wakens up to count waiters where (waitMask & wakeMask) != 0,
- * returning the number of awoken threads. */
+ /** Wakens up to count waiters where (waitMask & wakeMask) !=
+ * 0, returning the number of awoken threads, or -1 if an error
+ * occurred. Note that when constructing a concurrency primitive
+ * that can guard its own destruction, it is likely that you will
+ * want to ignore EINVAL here (as well as making sure that you
+ * never touch the object after performing the memory store that
+ * is the linearization point for unlock or control handoff).
+ * See https://sourceware.org/bugzilla/show_bug.cgi?id=13690 */
int futexWake(int count = std::numeric_limits<int>::max(),
uint32_t wakeMask = -1);
- private:
-
- /** Futex wait implemented via syscall SYS_futex. absTimeout gives
- * time till when the wait can block. If it is nullptr the call will
- * block until a matching futex wake is received. extraOpFlags can be
- * used to specify addtional flags to add to the futex operation (by
- * default only FUTEX_WAIT_BITSET and FUTEX_PRIVATE_FLAG are included).
- * Returns 0 on success or -1 on error, with errno set to one of the
- * values listed in futex(2). */
- int futexWaitImpl(uint32_t expected,
- const struct timespec* absTimeout,
- int extraOpFlags,
- uint32_t waitMask);
+ private:
+ /** Optimal when TargetClock is the same type as Clock.
+ *
+ * Otherwise, both Clock::now() and TargetClock::now() must be invoked. */
+ template <typename TargetClock, typename Clock, typename Duration>
+ static typename TargetClock::time_point time_point_conv(
+ std::chrono::time_point<Clock, Duration> const& time) {
+ using std::chrono::duration_cast;
+ using TargetDuration = typename TargetClock::duration;
+ using TargetTimePoint = typename TargetClock::time_point;
+ if (std::is_same<Clock, TargetClock>::value) {
+ // in place of time_point_cast, which cannot compile without if-constexpr
+ auto const delta = time.time_since_epoch();
+ return TargetTimePoint(duration_cast<TargetDuration>(delta));
+ } else {
+ // different clocks with different epochs, so non-optimal case
+ auto const delta = time - Clock::now();
+ return TargetClock::now() + duration_cast<TargetDuration>(delta);
+ }
+ }
+
+ template <typename Deadline>
+ typename std::enable_if<Deadline::clock::is_steady, FutexResult>::type
+ futexWaitImpl(
+ uint32_t expected,
+ Deadline const& deadline,
+ uint32_t waitMask) {
+ return futexWaitImpl(expected, nullptr, &deadline, waitMask);
+ }
+
+ template <typename Deadline>
+ typename std::enable_if<!Deadline::clock::is_steady, FutexResult>::type
+ futexWaitImpl(
+ uint32_t expected,
+ Deadline const& deadline,
+ uint32_t waitMask) {
+ return futexWaitImpl(expected, &deadline, nullptr, waitMask);
+ }
+
+ /** Underlying implementation of futexWait and futexWaitUntil.
+ * At most one of absSystemTime and absSteadyTime should be non-null.
+ * Timeouts are separated into separate parameters to allow the
+ * implementations to be elsewhere without templating on the clock
+ * type, which is otherwise complicated by the fact that steady_clock
+ * is the same as system_clock on some platforms. */
+ FutexResult futexWaitImpl(
+ uint32_t expected,
+ std::chrono::system_clock::time_point const* absSystemTime,
+ std::chrono::steady_clock::time_point const* absSteadyTime,
+ uint32_t waitMask);
};
-template <>
-inline int
-Futex<std::atomic>::futexWaitImpl(uint32_t expected,
- const struct timespec* absTimeout,
- int extraOpFlags,
- uint32_t waitMask) {
- assert(sizeof(*this) == sizeof(int));
-
- /* Unlike FUTEX_WAIT, FUTEX_WAIT_BITSET requires an absolute timeout
- * value - http://locklessinc.com/articles/futex_cheat_sheet/ */
- int rv = syscall(
- SYS_futex,
- this, /* addr1 */
- FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | extraOpFlags, /* op */
- expected, /* val */
- absTimeout, /* timeout */
- nullptr, /* addr2 */
- waitMask); /* val3 */
-
- assert(rv == 0 ||
- errno == EWOULDBLOCK ||
- errno == EINTR ||
- (absTimeout != nullptr && errno == ETIMEDOUT));
-
- return rv;
-}
+/** A std::atomic subclass that can be used to force Futex to emulate
+ * the underlying futex() syscall. This is primarily useful to test or
+ * benchmark the emulated implementation on systems that don't need it. */
+template <typename T>
+struct EmulatedFutexAtomic : public std::atomic<T> {
+ EmulatedFutexAtomic() noexcept = default;
+ constexpr /* implicit */ EmulatedFutexAtomic(T init) noexcept
+ : std::atomic<T>(init) {}
+ // It doesn't copy or move
+ EmulatedFutexAtomic(EmulatedFutexAtomic&& rhs) = delete;
+};
+
+/* Available specializations, with definitions elsewhere */
template <>
-inline bool Futex<std::atomic>::futexWait(uint32_t expected,
- uint32_t waitMask) {
- return futexWaitImpl(expected, nullptr, 0 /* extraOpFlags */, waitMask) == 0;
-}
+int Futex<std::atomic>::futexWake(int count, uint32_t wakeMask);
template <>
-inline int Futex<std::atomic>::futexWake(int count, uint32_t wakeMask) {
- assert(sizeof(*this) == sizeof(int));
- int rv = syscall(SYS_futex,
- this, /* addr1 */
- FUTEX_WAKE_BITSET | FUTEX_PRIVATE_FLAG, /* op */
- count, /* val */
- nullptr, /* timeout */
- nullptr, /* addr2 */
- wakeMask); /* val3 */
- assert(rv >= 0);
- return rv;
-}
-
-/* Convert std::chrono::time_point to struct timespec */
-template <class Clock, class Duration = typename Clock::Duration>
-struct timespec timePointToTimeSpec(const time_point<Clock, Duration>& tp) {
- using std::chrono::nanoseconds;
- using std::chrono::seconds;
- using std::chrono::duration_cast;
-
- struct timespec ts;
- auto duration = tp.time_since_epoch();
- auto secs = duration_cast<seconds>(duration);
- auto nanos = duration_cast<nanoseconds>(duration - secs);
- ts.tv_sec = secs.count();
- ts.tv_nsec = nanos.count();
- return ts;
-}
-
-template <template<typename> class Atom> template<class Clock, class Duration>
-inline FutexResult
-Futex<Atom>::futexWaitUntil(
- uint32_t expected,
- const time_point<Clock, Duration>& absTime,
- uint32_t waitMask) {
-
- static_assert(std::is_same<Clock,system_clock>::value ||
- std::is_same<Clock,steady_clock>::value,
- "Only std::system_clock or std::steady_clock supported");
-
- struct timespec absTimeSpec = timePointToTimeSpec(absTime);
- int extraOpFlags = 0;
-
- /* We must use FUTEX_CLOCK_REALTIME flag if we are getting the time_point
- * from the system clock (CLOCK_REALTIME). This check also works correctly for
- * broken glibc in which steady_clock is a typedef to system_clock.*/
- if (std::is_same<Clock,system_clock>::value) {
- extraOpFlags = FUTEX_CLOCK_REALTIME;
- } else {
- assert(Clock::is_steady);
- }
+FutexResult Futex<std::atomic>::futexWaitImpl(
+ uint32_t expected,
+ std::chrono::system_clock::time_point const* absSystemTime,
+ std::chrono::steady_clock::time_point const* absSteadyTime,
+ uint32_t waitMask);
- const int rv = futexWaitImpl(expected, &absTimeSpec, extraOpFlags, waitMask);
- return futexErrnoToFutexResult(rv, errno);
-}
+template <>
+int Futex<EmulatedFutexAtomic>::futexWake(int count, uint32_t wakeMask);
-}}
+template <>
+FutexResult Futex<EmulatedFutexAtomic>::futexWaitImpl(
+ uint32_t expected,
+ std::chrono::system_clock::time_point const* absSystemTime,
+ std::chrono::steady_clock::time_point const* absSteadyTime,
+ uint32_t waitMask);
+
+} // namespace detail
+} // namespace folly
projects / folly.git / blobdiff