X-Git-Url: http://plrg.eecs.uci.edu/git/?a=blobdiff_plain;f=folly%2FRWSpinLock.h;h=94f48028289ca496d6d29059ff3d2be0814643ab;hb=1a50cb78590fabf164630808f1be23a9bef6bcc7;hp=ae1aa6bff6da56cfb954a13b4de755929242f6bd;hpb=ab442ee8766aeac54c1bad394655a48c242c8607;p=folly.git diff --git a/folly/RWSpinLock.h b/folly/RWSpinLock.h index ae1aa6bf..94f48028 100644 --- a/folly/RWSpinLock.h +++ b/folly/RWSpinLock.h @@ -1,5 +1,5 @@ /* - * Copyright 2016 Facebook, Inc. + * Copyright 2017 Facebook, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -81,8 +81,7 @@ * @author Xin Liu */ -#ifndef FOLLY_RWSPINLOCK_H_ -#define FOLLY_RWSPINLOCK_H_ +#pragma once /* ======================================================================== @@ -137,30 +136,29 @@ pthread_rwlock_t Read 728698 24us 101ns 7.28ms 194us */ #include +#include -#if defined(__GNUC__) && \ - (defined(__i386) || FOLLY_X64 || \ - defined(ARCH_K8)) -# define RW_SPINLOCK_USE_X86_INTRINSIC_ -# include +#if defined(__GNUC__) && (defined(__i386) || FOLLY_X64 || defined(ARCH_K8)) +#define RW_SPINLOCK_USE_X86_INTRINSIC_ +#include #elif defined(_MSC_VER) && defined(FOLLY_X64) -# define RW_SPINLOCK_USE_X86_INTRINSIC_ +#define RW_SPINLOCK_USE_X86_INTRINSIC_ #else -# undef RW_SPINLOCK_USE_X86_INTRINSIC_ +#undef RW_SPINLOCK_USE_X86_INTRINSIC_ #endif // iOS doesn't define _mm_cvtsi64_si128 and friends -#if (FOLLY_SSE >= 2) && !TARGET_OS_IPHONE +#if (FOLLY_SSE >= 2) && !FOLLY_MOBILE && FOLLY_X64 #define RW_SPINLOCK_USE_SSE_INSTRUCTIONS_ #else #undef RW_SPINLOCK_USE_SSE_INSTRUCTIONS_ #endif +#include #include #include -#include +#include -#include #include #include @@ -192,9 +190,11 @@ class RWSpinLock { // Lockable Concept void lock() { - int count = 0; + uint_fast32_t count = 0; while (!LIKELY(try_lock())) { - if (++count > 1000) sched_yield(); + if (++count > 1000) { + std::this_thread::yield(); + } } } @@ -206,9 +206,11 @@ class RWSpinLock { // SharedLockable Concept void lock_shared() { - int count = 0; + uint_fast32_t count = 0; while (!LIKELY(try_lock_shared())) { - if (++count > 1000) sched_yield(); + if (++count > 1000) { + std::this_thread::yield(); + } } } @@ -224,9 +226,11 @@ class RWSpinLock { // UpgradeLockable Concept void lock_upgrade() { - int count = 0; + uint_fast32_t count = 0; while (!try_lock_upgrade()) { - if (++count > 1000) sched_yield(); + if (++count > 1000) { + std::this_thread::yield(); + } } } @@ -238,7 +242,9 @@ class RWSpinLock { void unlock_upgrade_and_lock() { int64_t count = 0; while (!try_unlock_upgrade_and_lock()) { - if (++count > 1000) sched_yield(); + if (++count > 1000) { + std::this_thread::yield(); + } } } @@ -307,8 +313,10 @@ class RWSpinLock { class ReadHolder { public: - explicit ReadHolder(RWSpinLock* lock = nullptr) : lock_(lock) { - if (lock_) lock_->lock_shared(); + explicit ReadHolder(RWSpinLock* lock) : lock_(lock) { + if (lock_) { + lock_->lock_shared(); + } } explicit ReadHolder(RWSpinLock& lock) : lock_(&lock) { @@ -322,12 +330,16 @@ class RWSpinLock { // down-grade explicit ReadHolder(UpgradedHolder&& upgraded) : lock_(upgraded.lock_) { upgraded.lock_ = nullptr; - if (lock_) lock_->unlock_upgrade_and_lock_shared(); + if (lock_) { + lock_->unlock_upgrade_and_lock_shared(); + } } explicit ReadHolder(WriteHolder&& writer) : lock_(writer.lock_) { writer.lock_ = nullptr; - if (lock_) lock_->unlock_and_lock_shared(); + if (lock_) { + lock_->unlock_and_lock_shared(); + } } ReadHolder& operator=(ReadHolder&& other) { @@ -339,13 +351,23 @@ class RWSpinLock { ReadHolder(const ReadHolder& other) = delete; ReadHolder& operator=(const ReadHolder& other) = delete; - ~ReadHolder() { if (lock_) lock_->unlock_shared(); } + ~ReadHolder() { + if (lock_) { + lock_->unlock_shared(); + } + } void reset(RWSpinLock* lock = nullptr) { - if (lock == lock_) return; - if (lock_) lock_->unlock_shared(); + if (lock == lock_) { + return; + } + if (lock_) { + lock_->unlock_shared(); + } lock_ = lock; - if (lock_) lock_->lock_shared(); + if (lock_) { + lock_->lock_shared(); + } } void swap(ReadHolder* other) { @@ -360,8 +382,10 @@ class RWSpinLock { class UpgradedHolder { public: - explicit UpgradedHolder(RWSpinLock* lock = nullptr) : lock_(lock) { - if (lock_) lock_->lock_upgrade(); + explicit UpgradedHolder(RWSpinLock* lock) : lock_(lock) { + if (lock_) { + lock_->lock_upgrade(); + } } explicit UpgradedHolder(RWSpinLock& lock) : lock_(&lock) { @@ -371,7 +395,9 @@ class RWSpinLock { explicit UpgradedHolder(WriteHolder&& writer) { lock_ = writer.lock_; writer.lock_ = nullptr; - if (lock_) lock_->unlock_and_lock_upgrade(); + if (lock_) { + lock_->unlock_and_lock_upgrade(); + } } UpgradedHolder(UpgradedHolder&& other) noexcept : lock_(other.lock_) { @@ -387,13 +413,23 @@ class RWSpinLock { UpgradedHolder(const UpgradedHolder& other) = delete; UpgradedHolder& operator =(const UpgradedHolder& other) = delete; - ~UpgradedHolder() { if (lock_) lock_->unlock_upgrade(); } + ~UpgradedHolder() { + if (lock_) { + lock_->unlock_upgrade(); + } + } void reset(RWSpinLock* lock = nullptr) { - if (lock == lock_) return; - if (lock_) lock_->unlock_upgrade(); + if (lock == lock_) { + return; + } + if (lock_) { + lock_->unlock_upgrade(); + } lock_ = lock; - if (lock_) lock_->lock_upgrade(); + if (lock_) { + lock_->lock_upgrade(); + } } void swap(UpgradedHolder* other) { @@ -409,8 +445,10 @@ class RWSpinLock { class WriteHolder { public: - explicit WriteHolder(RWSpinLock* lock = nullptr) : lock_(lock) { - if (lock_) lock_->lock(); + explicit WriteHolder(RWSpinLock* lock) : lock_(lock) { + if (lock_) { + lock_->lock(); + } } explicit WriteHolder(RWSpinLock& lock) : lock_(&lock) { @@ -421,7 +459,9 @@ class RWSpinLock { explicit WriteHolder(UpgradedHolder&& upgraded) { lock_ = upgraded.lock_; upgraded.lock_ = nullptr; - if (lock_) lock_->unlock_upgrade_and_lock(); + if (lock_) { + lock_->unlock_upgrade_and_lock(); + } } WriteHolder(WriteHolder&& other) noexcept : lock_(other.lock_) { @@ -437,13 +477,23 @@ class RWSpinLock { WriteHolder(const WriteHolder& other) = delete; WriteHolder& operator =(const WriteHolder& other) = delete; - ~WriteHolder () { if (lock_) lock_->unlock(); } + ~WriteHolder() { + if (lock_) { + lock_->unlock(); + } + } void reset(RWSpinLock* lock = nullptr) { - if (lock == lock_) return; - if (lock_) lock_->unlock(); + if (lock == lock_) { + return; + } + if (lock_) { + lock_->unlock(); + } lock_ = lock; - if (lock_) lock_->lock(); + if (lock_) { + lock_->lock(); + } } void swap(WriteHolder* other) { @@ -457,12 +507,6 @@ class RWSpinLock { RWSpinLock* lock_; }; - // Synchronized<> adaptors - friend void acquireRead(RWSpinLock& l) { return l.lock_shared(); } - friend void acquireReadWrite(RWSpinLock& l) { return l.lock(); } - friend void releaseRead(RWSpinLock& l) { return l.unlock_shared(); } - friend void releaseReadWrite(RWSpinLock& l) { return l.unlock(); } - private: std::atomic bits_; }; @@ -485,13 +529,14 @@ struct RWTicketIntTrait<64> { #ifdef RW_SPINLOCK_USE_SSE_INSTRUCTIONS_ static __m128i make128(const uint16_t v[4]) { - return _mm_set_epi16(0, 0, 0, 0, v[3], v[2], v[1], v[0]); + return _mm_set_epi16(0, 0, 0, 0, + short(v[3]), short(v[2]), short(v[1]), short(v[0])); } static inline __m128i fromInteger(uint64_t from) { - return _mm_cvtsi64_si128(from); + return _mm_cvtsi64_si128(int64_t(from)); } static inline uint64_t toInteger(__m128i in) { - return _mm_cvtsi128_si64(in); + return uint64_t(_mm_cvtsi128_si64(in)); } static inline uint64_t addParallel(__m128i in, __m128i kDelta) { return toInteger(_mm_add_epi16(in, kDelta)); @@ -507,24 +552,26 @@ struct RWTicketIntTrait<32> { #ifdef RW_SPINLOCK_USE_SSE_INSTRUCTIONS_ static __m128i make128(const uint8_t v[4]) { - return _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, v[3], v[2], v[1], v[0]); + return _mm_set_epi8( + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + char(v[3]), char(v[2]), char(v[1]), char(v[0])); } static inline __m128i fromInteger(uint32_t from) { - return _mm_cvtsi32_si128(from); + return _mm_cvtsi32_si128(int32_t(from)); } static inline uint32_t toInteger(__m128i in) { - return _mm_cvtsi128_si32(in); + return uint32_t(_mm_cvtsi128_si32(in)); } static inline uint32_t addParallel(__m128i in, __m128i kDelta) { return toInteger(_mm_add_epi8(in, kDelta)); } #endif }; -} // detail +} // namespace detail - -template +template class RWTicketSpinLockT { typedef detail::RWTicketIntTrait IntTraitType; typedef typename detail::RWTicketIntTrait::FullInt FullInt; @@ -544,13 +591,13 @@ class RWTicketSpinLockT { } ticket; private: // Some x64-specific utilities for atomic access to ticket. - template static T load_acquire(T* addr) { + template static T load_acquire(T* addr) { T t = *addr; // acquire barrier asm_volatile_memory(); return t; } - template + template static void store_release(T* addr, T v) { asm_volatile_memory(); *addr = v; // release barrier @@ -591,7 +638,9 @@ class RWTicketSpinLockT { bool try_lock() { RWTicket t; FullInt old = t.whole = load_acquire(&ticket.whole); - if (t.users != t.write) return false; + if (t.users != t.write) { + return false; + } ++t.users; return __sync_bool_compare_and_swap(&ticket.whole, old, t.whole); } @@ -603,16 +652,18 @@ class RWTicketSpinLockT { * turns. */ void writeLockAggressive() { - // sched_yield() is needed here to avoid a pathology if the number + // std::this_thread::yield() is needed here to avoid a pathology if the number // of threads attempting concurrent writes is >= the number of real // cores allocated to this process. This is less likely than the // corresponding situation in lock_shared(), but we still want to // avoid it - int count = 0; + uint_fast32_t count = 0; QuarterInt val = __sync_fetch_and_add(&ticket.users, 1); while (val != load_acquire(&ticket.write)) { asm_volatile_pause(); - if (UNLIKELY(++count > 1000)) sched_yield(); + if (UNLIKELY(++count > 1000)) { + std::this_thread::yield(); + } } } @@ -625,7 +676,7 @@ class RWTicketSpinLockT { // there are a lot of competing readers. The aggressive spinning // can help to avoid starving writers. // - // We don't worry about sched_yield() here because the caller + // We don't worry about std::this_thread::yield() here because the caller // has already explicitly abandoned fairness. while (!try_lock()) {} } @@ -655,13 +706,15 @@ class RWTicketSpinLockT { } void lock_shared() { - // sched_yield() is important here because we can't grab the + // std::this_thread::yield() is important here because we can't grab the // shared lock if there is a pending writeLockAggressive, so we // need to let threads that already have a shared lock complete - int count = 0; + uint_fast32_t count = 0; while (!LIKELY(try_lock_shared())) { asm_volatile_pause(); - if (UNLIKELY((++count & 1023) == 0)) sched_yield(); + if (UNLIKELY((++count & 1023) == 0)) { + std::this_thread::yield(); + } } } @@ -683,7 +736,7 @@ class RWTicketSpinLockT { } void unlock_shared() { - QuarterInt val = __sync_fetch_and_add(&ticket.write, 1); + __sync_fetch_and_add(&ticket.write, 1); } class WriteHolder; @@ -694,13 +747,16 @@ class RWTicketSpinLockT { ReadHolder(ReadHolder const&) = delete; ReadHolder& operator=(ReadHolder const&) = delete; - explicit ReadHolder(RWSpinLock *lock = nullptr) : - lock_(lock) { - if (lock_) lock_->lock_shared(); + explicit ReadHolder(RWSpinLock* lock) : lock_(lock) { + if (lock_) { + lock_->lock_shared(); + } } explicit ReadHolder(RWSpinLock &lock) : lock_ (&lock) { - if (lock_) lock_->lock_shared(); + if (lock_) { + lock_->lock_shared(); + } } // atomically unlock the write-lock from writer and acquire the read-lock @@ -712,13 +768,19 @@ class RWTicketSpinLockT { } ~ReadHolder() { - if (lock_) lock_->unlock_shared(); + if (lock_) { + lock_->unlock_shared(); + } } void reset(RWSpinLock *lock = nullptr) { - if (lock_) lock_->unlock_shared(); + if (lock_) { + lock_->unlock_shared(); + } lock_ = lock; - if (lock_) lock_->lock_shared(); + if (lock_) { + lock_->lock_shared(); + } } void swap(ReadHolder *other) { @@ -734,22 +796,34 @@ class RWTicketSpinLockT { WriteHolder(WriteHolder const&) = delete; WriteHolder& operator=(WriteHolder const&) = delete; - explicit WriteHolder(RWSpinLock *lock = nullptr) : lock_(lock) { - if (lock_) lock_->lock(); + explicit WriteHolder(RWSpinLock* lock) : lock_(lock) { + if (lock_) { + lock_->lock(); + } } explicit WriteHolder(RWSpinLock &lock) : lock_ (&lock) { - if (lock_) lock_->lock(); + if (lock_) { + lock_->lock(); + } } ~WriteHolder() { - if (lock_) lock_->unlock(); + if (lock_) { + lock_->unlock(); + } } void reset(RWSpinLock *lock = nullptr) { - if (lock == lock_) return; - if (lock_) lock_->unlock(); + if (lock == lock_) { + return; + } + if (lock_) { + lock_->unlock(); + } lock_ = lock; - if (lock_) lock_->lock(); + if (lock_) { + lock_->lock(); + } } void swap(WriteHolder *other) { @@ -760,20 +834,6 @@ class RWTicketSpinLockT { friend class ReadHolder; RWSpinLock *lock_; }; - - // Synchronized<> adaptors. - friend void acquireRead(RWTicketSpinLockT& mutex) { - mutex.lock_shared(); - } - friend void acquireReadWrite(RWTicketSpinLockT& mutex) { - mutex.lock(); - } - friend void releaseRead(RWTicketSpinLockT& mutex) { - mutex.unlock_shared(); - } - friend void releaseReadWrite(RWTicketSpinLockT& mutex) { - mutex.unlock(); - } }; typedef RWTicketSpinLockT<32> RWTicketSpinLock32; @@ -781,10 +841,8 @@ typedef RWTicketSpinLockT<64> RWTicketSpinLock64; #endif // RW_SPINLOCK_USE_X86_INTRINSIC_ -} // namespace folly +} // namespace folly #ifdef RW_SPINLOCK_USE_X86_INTRINSIC_ #undef RW_SPINLOCK_USE_X86_INTRINSIC_ #endif - -#endif // FOLLY_RWSPINLOCK_H_