folly/test/SharedMutexTest.cpp

   1 /*
   2  * Copyright 2015 Facebook, Inc.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *   http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #include <folly/SharedMutex.h>
  18
  19 #include <stdlib.h>
  20 #include <thread>
  21 #include <vector>
  22 #include <boost/optional.hpp>
  23 #include <folly/Benchmark.h>
  24 #include <folly/MPMCQueue.h>
  25 #include <folly/Random.h>
  26 #include <folly/test/DeterministicSchedule.h>
  27 #include <gflags/gflags.h>
  28 #include <gtest/gtest.h>
  29
  30 #include <boost/thread/shared_mutex.hpp>
  31 #include <folly/RWSpinLock.h>
  32
  33 using namespace folly;
  34 using namespace folly::test;
  35 using namespace std;
  36 using namespace chrono;
  37
  38 typedef DeterministicSchedule DSched;
  39 typedef SharedMutexImpl<true, void, DeterministicAtomic, true>
  40     DSharedMutexReadPriority;
  41 typedef SharedMutexImpl<false, void, DeterministicAtomic, true>
  42     DSharedMutexWritePriority;
  43
  44 COMMON_CONCURRENCY_SHARED_MUTEX_DECLARE_STATIC_STORAGE(
  45     DSharedMutexReadPriority);
  46 COMMON_CONCURRENCY_SHARED_MUTEX_DECLARE_STATIC_STORAGE(
  47     DSharedMutexWritePriority);
  48
  49 template <typename Lock>
  50 void runBasicTest() {
  51   Lock lock;
  52   SharedMutexToken token1;
  53   SharedMutexToken token2;
  54   SharedMutexToken token3;
  55
  56   EXPECT_TRUE(lock.try_lock());
  57   EXPECT_FALSE(lock.try_lock());
  58   EXPECT_FALSE(lock.try_lock_shared(token1));
  59   lock.unlock();
  60
  61   EXPECT_TRUE(lock.try_lock_shared(token1));
  62   EXPECT_FALSE(lock.try_lock());
  63   EXPECT_TRUE(lock.try_lock_shared(token2));
  64   lock.lock_shared(token3);
  65   lock.unlock_shared(token3);
  66   lock.unlock_shared(token2);
  67   lock.unlock_shared(token1);
  68
  69   lock.lock();
  70   lock.unlock();
  71
  72   lock.lock_shared(token1);
  73   lock.lock_shared(token2);
  74   lock.unlock_shared(token1);
  75   lock.unlock_shared(token2);
  76
  77   lock.lock();
  78   lock.unlock_and_lock_shared(token1);
  79   lock.lock_shared(token2);
  80   lock.unlock_shared(token2);
  81   lock.unlock_shared(token1);
  82 }
  83
  84 TEST(SharedMutex, basic) {
  85   runBasicTest<SharedMutexReadPriority>();
  86   runBasicTest<SharedMutexWritePriority>();
  87 }
  88
  89 template <typename Lock>
  90 void runBasicHoldersTest() {
  91   Lock lock;
  92   SharedMutexToken token;
  93
  94   {
  95     typename Lock::WriteHolder holder(lock);
  96     EXPECT_FALSE(lock.try_lock());
  97     EXPECT_FALSE(lock.try_lock_shared(token));
  98
  99     typename Lock::WriteHolder holder2(std::move(holder));
 100     typename Lock::WriteHolder holder3;
 101     holder3 = std::move(holder2);
 102
 103     typename Lock::UpgradeHolder holder4(std::move(holder3));
 104     typename Lock::WriteHolder holder5(std::move(holder4));
 105
 106     typename Lock::ReadHolder holder6(std::move(holder5));
 107
 108     EXPECT_FALSE(lock.try_lock());
 109     EXPECT_TRUE(lock.try_lock_shared(token));
 110     lock.unlock_shared(token);
 111   }
 112
 113   {
 114     typename Lock::WriteHolder holder(lock);
 115     EXPECT_FALSE(lock.try_lock());
 116   }
 117
 118   {
 119     typename Lock::ReadHolder holder(lock);
 120     typename Lock::ReadHolder holder2(lock);
 121     typename Lock::UpgradeHolder holder3(lock);
 122   }
 123
 124   {
 125     typename Lock::UpgradeHolder holder(lock);
 126     typename Lock::ReadHolder holder2(lock);
 127     typename Lock::ReadHolder holder3(std::move(holder));
 128   }
 129 }
 130
 131 TEST(SharedMutex, basic_holders) {
 132   runBasicHoldersTest<SharedMutexReadPriority>();
 133   runBasicHoldersTest<SharedMutexWritePriority>();
 134 }
 135
 136 template <typename Lock>
 137 void runManyReadLocksTestWithTokens() {
 138   Lock lock;
 139
 140   vector<SharedMutexToken> tokens;
 141   for (int i = 0; i < 1000; ++i) {
 142     tokens.emplace_back();
 143     EXPECT_TRUE(lock.try_lock_shared(tokens.back()));
 144   }
 145   for (auto& token : tokens) {
 146     lock.unlock_shared(token);
 147   }
 148   EXPECT_TRUE(lock.try_lock());
 149   lock.unlock();
 150 }
 151
 152 TEST(SharedMutex, many_read_locks_with_tokens) {
 153   runManyReadLocksTestWithTokens<SharedMutexReadPriority>();
 154   runManyReadLocksTestWithTokens<SharedMutexWritePriority>();
 155 }
 156
 157 template <typename Lock>
 158 void runManyReadLocksTestWithoutTokens() {
 159   Lock lock;
 160
 161   for (int i = 0; i < 1000; ++i) {
 162     EXPECT_TRUE(lock.try_lock_shared());
 163   }
 164   for (int i = 0; i < 1000; ++i) {
 165     lock.unlock_shared();
 166   }
 167   EXPECT_TRUE(lock.try_lock());
 168   lock.unlock();
 169 }
 170
 171 TEST(SharedMutex, many_read_locks_without_tokens) {
 172   runManyReadLocksTestWithoutTokens<SharedMutexReadPriority>();
 173   runManyReadLocksTestWithoutTokens<SharedMutexWritePriority>();
 174 }
 175
 176 template <typename Lock>
 177 void runTimeoutInPastTest() {
 178   Lock lock;
 179
 180   EXPECT_TRUE(lock.try_lock_for(milliseconds(0)));
 181   lock.unlock();
 182   EXPECT_TRUE(lock.try_lock_for(milliseconds(-1)));
 183   lock.unlock();
 184   EXPECT_TRUE(lock.try_lock_shared_for(milliseconds(0)));
 185   lock.unlock_shared();
 186   EXPECT_TRUE(lock.try_lock_shared_for(milliseconds(-1)));
 187   lock.unlock_shared();
 188   EXPECT_TRUE(lock.try_lock_until(system_clock::now() - milliseconds(1)));
 189   lock.unlock();
 190   EXPECT_TRUE(
 191       lock.try_lock_shared_until(system_clock::now() - milliseconds(1)));
 192   lock.unlock_shared();
 193   EXPECT_TRUE(lock.try_lock_until(steady_clock::now() - milliseconds(1)));
 194   lock.unlock();
 195   EXPECT_TRUE(
 196       lock.try_lock_shared_until(steady_clock::now() - milliseconds(1)));
 197   lock.unlock_shared();
 198 }
 199
 200 TEST(SharedMutex, timeout_in_past) {
 201   runTimeoutInPastTest<SharedMutexReadPriority>();
 202   runTimeoutInPastTest<SharedMutexWritePriority>();
 203 }
 204
 205 template <class Func>
 206 bool funcHasDuration(milliseconds expectedDuration, Func func) {
 207   // elapsed time should eventually fall within expectedDuration +- 25%
 208   for (int tries = 0; tries < 100; ++tries) {
 209     auto start = steady_clock::now();
 210     func();
 211     auto elapsed = steady_clock::now() - start;
 212     if (elapsed > expectedDuration - expectedDuration / 4 &&
 213         elapsed < expectedDuration + expectedDuration / 4) {
 214       return true;
 215     }
 216   }
 217   return false;
 218 }
 219
 220 template <typename Lock>
 221 void runFailingTryTimeoutTest() {
 222   Lock lock;
 223   lock.lock();
 224   EXPECT_TRUE(funcHasDuration(milliseconds(10), [&] {
 225     EXPECT_FALSE(lock.try_lock_for(milliseconds(10)));
 226   }));
 227   EXPECT_TRUE(funcHasDuration(milliseconds(10), [&] {
 228     typename Lock::Token token;
 229     EXPECT_FALSE(lock.try_lock_shared_for(milliseconds(10), token));
 230   }));
 231   EXPECT_TRUE(funcHasDuration(milliseconds(10), [&] {
 232     EXPECT_FALSE(lock.try_lock_upgrade_for(milliseconds(10)));
 233   }));
 234   EXPECT_TRUE(funcHasDuration(milliseconds(10), [&] {
 235     EXPECT_FALSE(lock.try_lock_until(steady_clock::now() + milliseconds(10)));
 236   }));
 237   EXPECT_TRUE(funcHasDuration(milliseconds(10), [&] {
 238     typename Lock::Token token;
 239     EXPECT_FALSE(lock.try_lock_shared_until(
 240         steady_clock::now() + milliseconds(10), token));
 241   }));
 242   EXPECT_TRUE(funcHasDuration(milliseconds(10), [&] {
 243     EXPECT_FALSE(
 244         lock.try_lock_upgrade_until(steady_clock::now() + milliseconds(10)));
 245   }));
 246   EXPECT_TRUE(funcHasDuration(milliseconds(10), [&] {
 247     EXPECT_FALSE(lock.try_lock_until(system_clock::now() + milliseconds(10)));
 248   }));
 249   EXPECT_TRUE(funcHasDuration(milliseconds(10), [&] {
 250     typename Lock::Token token;
 251     EXPECT_FALSE(lock.try_lock_shared_until(
 252         system_clock::now() + milliseconds(10), token));
 253   }));
 254   EXPECT_TRUE(funcHasDuration(milliseconds(10), [&] {
 255     EXPECT_FALSE(
 256         lock.try_lock_upgrade_until(system_clock::now() + milliseconds(10)));
 257   }));
 258   lock.unlock();
 259
 260   lock.lock_shared();
 261   EXPECT_TRUE(funcHasDuration(milliseconds(10), [&] {
 262     EXPECT_FALSE(lock.try_lock_for(milliseconds(10)));
 263   }));
 264   EXPECT_TRUE(funcHasDuration(milliseconds(10), [&] {
 265     EXPECT_FALSE(lock.try_lock_until(steady_clock::now() + milliseconds(10)));
 266   }));
 267   EXPECT_TRUE(funcHasDuration(milliseconds(10), [&] {
 268     EXPECT_FALSE(lock.try_lock_until(system_clock::now() + milliseconds(10)));
 269   }));
 270   lock.unlock_shared();
 271
 272   lock.lock();
 273   for (int p = 0; p < 8; ++p) {
 274     EXPECT_FALSE(lock.try_lock_for(nanoseconds(1 << p)));
 275   }
 276   lock.unlock();
 277
 278   for (int p = 0; p < 8; ++p) {
 279     typename Lock::ReadHolder holder1(lock);
 280     typename Lock::ReadHolder holder2(lock);
 281     typename Lock::ReadHolder holder3(lock);
 282     EXPECT_FALSE(lock.try_lock_for(nanoseconds(1 << p)));
 283   }
 284 }
 285
 286 TEST(SharedMutex, failing_try_timeout) {
 287   runFailingTryTimeoutTest<SharedMutexReadPriority>();
 288   runFailingTryTimeoutTest<SharedMutexWritePriority>();
 289 }
 290
 291 template <typename Lock>
 292 void runBasicUpgradeTest() {
 293   Lock lock;
 294   typename Lock::Token token1;
 295   typename Lock::Token token2;
 296
 297   lock.lock_upgrade();
 298   EXPECT_FALSE(lock.try_lock());
 299   EXPECT_TRUE(lock.try_lock_shared(token1));
 300   lock.unlock_shared(token1);
 301   lock.unlock_upgrade();
 302
 303   lock.lock_upgrade();
 304   lock.unlock_upgrade_and_lock();
 305   EXPECT_FALSE(lock.try_lock_shared(token1));
 306   lock.unlock();
 307
 308   lock.lock_upgrade();
 309   lock.unlock_upgrade_and_lock_shared(token1);
 310   lock.lock_upgrade();
 311   lock.unlock_upgrade_and_lock_shared(token2);
 312   lock.unlock_shared(token1);
 313   lock.unlock_shared(token2);
 314
 315   lock.lock();
 316   lock.unlock_and_lock_upgrade();
 317   EXPECT_TRUE(lock.try_lock_shared(token1));
 318   lock.unlock_upgrade();
 319   lock.unlock_shared(token1);
 320 }
 321
 322 TEST(SharedMutex, basic_upgrade_tests) {
 323   runBasicUpgradeTest<SharedMutexReadPriority>();
 324   runBasicUpgradeTest<SharedMutexWritePriority>();
 325 }
 326
 327 TEST(SharedMutex, read_has_prio) {
 328   SharedMutexReadPriority lock;
 329   SharedMutexToken token1;
 330   SharedMutexToken token2;
 331   lock.lock_shared(token1);
 332   bool exclusiveAcquired = false;
 333   auto writer = thread([&] {
 334     lock.lock();
 335     exclusiveAcquired = true;
 336     lock.unlock();
 337   });
 338
 339   // lock() can't complete until we unlock token1, but it should stake
 340   // its claim with regards to other exclusive or upgrade locks.  We can
 341   // use try_lock_upgrade to poll for that eventuality.
 342   while (lock.try_lock_upgrade()) {
 343     lock.unlock_upgrade();
 344     this_thread::yield();
 345   }
 346   EXPECT_FALSE(exclusiveAcquired);
 347
 348   // Even though lock() is stuck we should be able to get token2
 349   EXPECT_TRUE(lock.try_lock_shared(token2));
 350   lock.unlock_shared(token1);
 351   lock.unlock_shared(token2);
 352   writer.join();
 353   EXPECT_TRUE(exclusiveAcquired);
 354 }
 355
 356 TEST(SharedMutex, write_has_prio) {
 357   SharedMutexWritePriority lock;
 358   SharedMutexToken token1;
 359   SharedMutexToken token2;
 360   lock.lock_shared(token1);
 361   auto writer = thread([&] {
 362     lock.lock();
 363     lock.unlock();
 364   });
 365
 366   // eventually lock() should block readers
 367   while (lock.try_lock_shared(token2)) {
 368     lock.unlock_shared(token2);
 369     this_thread::yield();
 370   }
 371
 372   lock.unlock_shared(token1);
 373   writer.join();
 374 }
 375
 376 struct TokenLocker {
 377   SharedMutexToken token;
 378
 379   template <typename T>
 380   void lock(T* lock) {
 381     lock->lock();
 382   }
 383
 384   template <typename T>
 385   void unlock(T* lock) {
 386     lock->unlock();
 387   }
 388
 389   template <typename T>
 390   void lock_shared(T* lock) {
 391     lock->lock_shared(token);
 392   }
 393
 394   template <typename T>
 395   void unlock_shared(T* lock) {
 396     lock->unlock_shared(token);
 397   }
 398 };
 399
 400 struct Locker {
 401   template <typename T>
 402   void lock(T* lock) {
 403     lock->lock();
 404   }
 405
 406   template <typename T>
 407   void unlock(T* lock) {
 408     lock->unlock();
 409   }
 410
 411   template <typename T>
 412   void lock_shared(T* lock) {
 413     lock->lock_shared();
 414   }
 415
 416   template <typename T>
 417   void unlock_shared(T* lock) {
 418     lock->unlock_shared();
 419   }
 420 };
 421
 422 struct EnterLocker {
 423   template <typename T>
 424   void lock(T* lock) {
 425     lock->lock(0);
 426   }
 427
 428   template <typename T>
 429   void unlock(T* lock) {
 430     lock->unlock();
 431   }
 432
 433   template <typename T>
 434   void lock_shared(T* lock) {
 435     lock->enter(0);
 436   }
 437
 438   template <typename T>
 439   void unlock_shared(T* lock) {
 440     lock->leave();
 441   }
 442 };
 443
 444 struct PosixRWLock {
 445   pthread_rwlock_t lock_;
 446
 447   PosixRWLock() { pthread_rwlock_init(&lock_, nullptr); }
 448
 449   ~PosixRWLock() { pthread_rwlock_destroy(&lock_); }
 450
 451   void lock() { pthread_rwlock_wrlock(&lock_); }
 452
 453   void unlock() { pthread_rwlock_unlock(&lock_); }
 454
 455   void lock_shared() { pthread_rwlock_rdlock(&lock_); }
 456
 457   void unlock_shared() { pthread_rwlock_unlock(&lock_); }
 458 };
 459
 460 struct PosixMutex {
 461   pthread_mutex_t lock_;
 462
 463   PosixMutex() { pthread_mutex_init(&lock_, nullptr); }
 464
 465   ~PosixMutex() { pthread_mutex_destroy(&lock_); }
 466
 467   void lock() { pthread_mutex_lock(&lock_); }
 468
 469   void unlock() { pthread_mutex_unlock(&lock_); }
 470
 471   void lock_shared() { pthread_mutex_lock(&lock_); }
 472
 473   void unlock_shared() { pthread_mutex_unlock(&lock_); }
 474 };
 475
 476 template <template <typename> class Atom, typename Lock, typename Locker>
 477 static void runContendedReaders(size_t numOps,
 478                                 size_t numThreads,
 479                                 bool useSeparateLocks) {
 480   char padding1[64];
 481   Lock globalLock;
 482   int valueProtectedByLock = 10;
 483   char padding2[64];
 484   Atom<bool> go(false);
 485   Atom<bool>* goPtr = &go; // workaround for clang bug
 486   vector<thread> threads(numThreads);
 487
 488   BENCHMARK_SUSPEND {
 489     for (size_t t = 0; t < numThreads; ++t) {
 490       threads[t] = DSched::thread([&, t, numThreads] {
 491         Lock privateLock;
 492         Lock* lock = useSeparateLocks ? &privateLock : &globalLock;
 493         Locker locker;
 494         while (!goPtr->load()) {
 495           this_thread::yield();
 496         }
 497         for (size_t op = t; op < numOps; op += numThreads) {
 498           locker.lock_shared(lock);
 499           // note: folly::doNotOptimizeAway reads and writes to its arg,
 500           // so the following two lines are very different than a call
 501           // to folly::doNotOptimizeAway(valueProtectedByLock);
 502           auto copy = valueProtectedByLock;
 503           folly::doNotOptimizeAway(copy);
 504           locker.unlock_shared(lock);
 505         }
 506       });
 507     }
 508   }
 509
 510   go.store(true);
 511   for (auto& thr : threads) {
 512     DSched::join(thr);
 513   }
 514 }
 515
 516 static void folly_rwspin_reads(uint numOps,
 517                                size_t numThreads,
 518                                bool useSeparateLocks) {
 519   runContendedReaders<atomic, RWSpinLock, Locker>(
 520       numOps, numThreads, useSeparateLocks);
 521 }
 522
 523 static void shmtx_wr_pri_reads(uint numOps,
 524                                size_t numThreads,
 525                                bool useSeparateLocks) {
 526   runContendedReaders<atomic, SharedMutexWritePriority, TokenLocker>(
 527       numOps, numThreads, useSeparateLocks);
 528 }
 529
 530 static void shmtx_w_bare_reads(uint numOps,
 531                                size_t numThreads,
 532                                bool useSeparateLocks) {
 533   runContendedReaders<atomic, SharedMutexWritePriority, Locker>(
 534       numOps, numThreads, useSeparateLocks);
 535 }
 536
 537 static void shmtx_rd_pri_reads(uint numOps,
 538                                size_t numThreads,
 539                                bool useSeparateLocks) {
 540   runContendedReaders<atomic, SharedMutexReadPriority, TokenLocker>(
 541       numOps, numThreads, useSeparateLocks);
 542 }
 543
 544 static void shmtx_r_bare_reads(uint numOps,
 545                                size_t numThreads,
 546                                bool useSeparateLocks) {
 547   runContendedReaders<atomic, SharedMutexReadPriority, Locker>(
 548       numOps, numThreads, useSeparateLocks);
 549 }
 550
 551 static void folly_ticket_reads(uint numOps,
 552                                size_t numThreads,
 553                                bool useSeparateLocks) {
 554   runContendedReaders<atomic, RWTicketSpinLock64, Locker>(
 555       numOps, numThreads, useSeparateLocks);
 556 }
 557
 558 static void boost_shared_reads(uint numOps,
 559                                size_t numThreads,
 560                                bool useSeparateLocks) {
 561   runContendedReaders<atomic, boost::shared_mutex, Locker>(
 562       numOps, numThreads, useSeparateLocks);
 563 }
 564
 565 static void pthrd_rwlock_reads(uint numOps,
 566                                size_t numThreads,
 567                                bool useSeparateLocks) {
 568   runContendedReaders<atomic, PosixRWLock, Locker>(
 569       numOps, numThreads, useSeparateLocks);
 570 }
 571
 572 template <template <typename> class Atom, typename Lock, typename Locker>
 573 static void runMixed(size_t numOps,
 574                      size_t numThreads,
 575                      double writeFraction,
 576                      bool useSeparateLocks) {
 577   char padding1[64];
 578   Lock globalLock;
 579   int valueProtectedByLock = 0;
 580   char padding2[64];
 581   Atom<bool> go(false);
 582   Atom<bool>* goPtr = &go; // workaround for clang bug
 583   vector<thread> threads(numThreads);
 584
 585   BENCHMARK_SUSPEND {
 586     for (size_t t = 0; t < numThreads; ++t) {
 587       threads[t] = DSched::thread([&, t, numThreads] {
 588         struct drand48_data buffer;
 589         srand48_r(t, &buffer);
 590         long writeThreshold = writeFraction * 0x7fffffff;
 591         Lock privateLock;
 592         Lock* lock = useSeparateLocks ? &privateLock : &globalLock;
 593         Locker locker;
 594         while (!goPtr->load()) {
 595           this_thread::yield();
 596         }
 597         for (size_t op = t; op < numOps; op += numThreads) {
 598           long randVal;
 599           lrand48_r(&buffer, &randVal);
 600           bool writeOp = randVal < writeThreshold;
 601           SharedMutexToken token;
 602           if (writeOp) {
 603             locker.lock(lock);
 604             if (!useSeparateLocks) {
 605               ++valueProtectedByLock;
 606             }
 607             locker.unlock(lock);
 608           } else {
 609             locker.lock_shared(lock);
 610             auto v = valueProtectedByLock;
 611             folly::doNotOptimizeAway(v);
 612             locker.unlock_shared(lock);
 613           }
 614         }
 615       });
 616     }
 617   }
 618
 619   go.store(true);
 620   for (auto& thr : threads) {
 621     DSched::join(thr);
 622   }
 623 }
 624
 625 static void folly_rwspin(size_t numOps,
 626                          size_t numThreads,
 627                          double writeFraction,
 628                          bool useSeparateLocks) {
 629   runMixed<atomic, RWSpinLock, Locker>(
 630       numOps, numThreads, writeFraction, useSeparateLocks);
 631 }
 632
 633 static void shmtx_wr_pri(uint numOps,
 634                          size_t numThreads,
 635                          double writeFraction,
 636                          bool useSeparateLocks) {
 637   runMixed<atomic, SharedMutexWritePriority, TokenLocker>(
 638       numOps, numThreads, writeFraction, useSeparateLocks);
 639 }
 640
 641 static void shmtx_w_bare(uint numOps,
 642                          size_t numThreads,
 643                          double writeFraction,
 644                          bool useSeparateLocks) {
 645   runMixed<atomic, SharedMutexWritePriority, Locker>(
 646       numOps, numThreads, writeFraction, useSeparateLocks);
 647 }
 648
 649 static void shmtx_rd_pri(uint numOps,
 650                          size_t numThreads,
 651                          double writeFraction,
 652                          bool useSeparateLocks) {
 653   runMixed<atomic, SharedMutexReadPriority, TokenLocker>(
 654       numOps, numThreads, writeFraction, useSeparateLocks);
 655 }
 656
 657 static void shmtx_r_bare(uint numOps,
 658                          size_t numThreads,
 659                          double writeFraction,
 660                          bool useSeparateLocks) {
 661   runMixed<atomic, SharedMutexReadPriority, Locker>(
 662       numOps, numThreads, writeFraction, useSeparateLocks);
 663 }
 664
 665 static void folly_ticket(size_t numOps,
 666                          size_t numThreads,
 667                          double writeFraction,
 668                          bool useSeparateLocks) {
 669   runMixed<atomic, RWTicketSpinLock64, Locker>(
 670       numOps, numThreads, writeFraction, useSeparateLocks);
 671 }
 672
 673 static void boost_shared(size_t numOps,
 674                          size_t numThreads,
 675                          double writeFraction,
 676                          bool useSeparateLocks) {
 677   runMixed<atomic, boost::shared_mutex, Locker>(
 678       numOps, numThreads, writeFraction, useSeparateLocks);
 679 }
 680
 681 static void pthrd_rwlock(size_t numOps,
 682                          size_t numThreads,
 683                          double writeFraction,
 684                          bool useSeparateLocks) {
 685   runMixed<atomic, PosixRWLock, Locker>(
 686       numOps, numThreads, writeFraction, useSeparateLocks);
 687 }
 688
 689 static void pthrd_mutex_(size_t numOps,
 690                          size_t numThreads,
 691                          double writeFraction,
 692                          bool useSeparateLocks) {
 693   runMixed<atomic, PosixMutex, Locker>(
 694       numOps, numThreads, writeFraction, useSeparateLocks);
 695 }
 696
 697 template <typename Lock, template <typename> class Atom>
 698 static void runAllAndValidate(size_t numOps, size_t numThreads) {
 699   Lock globalLock;
 700   Atom<int> globalExclusiveCount(0);
 701   Atom<int> globalUpgradeCount(0);
 702   Atom<int> globalSharedCount(0);
 703
 704   Atom<bool> go(false);
 705
 706   // clang crashes on access to Atom<> captured by ref in closure
 707   Atom<int>* globalExclusiveCountPtr = &globalExclusiveCount;
 708   Atom<int>* globalUpgradeCountPtr = &globalUpgradeCount;
 709   Atom<int>* globalSharedCountPtr = &globalSharedCount;
 710   Atom<bool>* goPtr = &go;
 711
 712   vector<thread> threads(numThreads);
 713
 714   BENCHMARK_SUSPEND {
 715     for (size_t t = 0; t < numThreads; ++t) {
 716       threads[t] = DSched::thread([&, t, numThreads] {
 717         struct drand48_data buffer;
 718         srand48_r(t, &buffer);
 719
 720         bool exclusive = false;
 721         bool upgrade = false;
 722         bool shared = false;
 723         bool ourGlobalTokenUsed = false;
 724         SharedMutexToken ourGlobalToken;
 725
 726         Lock privateLock;
 727         vector<SharedMutexToken> privateTokens;
 728
 729         while (!goPtr->load()) {
 730           this_thread::yield();
 731         }
 732         for (size_t op = t; op < numOps; op += numThreads) {
 733           // randVal in [0,1000)
 734           long randVal;
 735           lrand48_r(&buffer, &randVal);
 736           randVal = (long)((randVal * (uint64_t)1000) / 0x7fffffff);
 737
 738           // make as many assertions as possible about the global state
 739           if (exclusive) {
 740             EXPECT_EQ(1, globalExclusiveCountPtr->load(memory_order_acquire));
 741             EXPECT_EQ(0, globalUpgradeCountPtr->load(memory_order_acquire));
 742             EXPECT_EQ(0, globalSharedCountPtr->load(memory_order_acquire));
 743           }
 744           if (upgrade) {
 745             EXPECT_EQ(0, globalExclusiveCountPtr->load(memory_order_acquire));
 746             EXPECT_EQ(1, globalUpgradeCountPtr->load(memory_order_acquire));
 747           }
 748           if (shared) {
 749             EXPECT_EQ(0, globalExclusiveCountPtr->load(memory_order_acquire));
 750             EXPECT_TRUE(globalSharedCountPtr->load(memory_order_acquire) > 0);
 751           } else {
 752             EXPECT_FALSE(ourGlobalTokenUsed);
 753           }
 754
 755           // independent 20% chance we do something to the private lock
 756           if (randVal < 200) {
 757             // it's okay to take multiple private shared locks because
 758             // we never take an exclusive lock, so reader versus writer
 759             // priority doesn't cause deadlocks
 760             if (randVal < 100 && privateTokens.size() > 0) {
 761               auto i = randVal % privateTokens.size();
 762               privateLock.unlock_shared(privateTokens[i]);
 763               privateTokens.erase(privateTokens.begin() + i);
 764             } else {
 765               SharedMutexToken token;
 766               privateLock.lock_shared(token);
 767               privateTokens.push_back(token);
 768             }
 769             continue;
 770           }
 771
 772           // if we've got a lock, the only thing we can do is release it
 773           // or transform it into a different kind of lock
 774           if (exclusive) {
 775             exclusive = false;
 776             --*globalExclusiveCountPtr;
 777             if (randVal < 500) {
 778               globalLock.unlock();
 779             } else if (randVal < 700) {
 780               globalLock.unlock_and_lock_shared();
 781               ++*globalSharedCountPtr;
 782               shared = true;
 783             } else if (randVal < 900) {
 784               globalLock.unlock_and_lock_shared(ourGlobalToken);
 785               ++*globalSharedCountPtr;
 786               shared = true;
 787               ourGlobalTokenUsed = true;
 788             } else {
 789               globalLock.unlock_and_lock_upgrade();
 790               ++*globalUpgradeCountPtr;
 791               upgrade = true;
 792             }
 793           } else if (upgrade) {
 794             upgrade = false;
 795             --*globalUpgradeCountPtr;
 796             if (randVal < 500) {
 797               globalLock.unlock_upgrade();
 798             } else if (randVal < 700) {
 799               globalLock.unlock_upgrade_and_lock_shared();
 800               ++*globalSharedCountPtr;
 801               shared = true;
 802             } else if (randVal < 900) {
 803               globalLock.unlock_upgrade_and_lock_shared(ourGlobalToken);
 804               ++*globalSharedCountPtr;
 805               shared = true;
 806               ourGlobalTokenUsed = true;
 807             } else {
 808               globalLock.unlock_upgrade_and_lock();
 809               ++*globalExclusiveCountPtr;
 810               exclusive = true;
 811             }
 812           } else if (shared) {
 813             shared = false;
 814             --*globalSharedCountPtr;
 815             if (ourGlobalTokenUsed) {
 816               globalLock.unlock_shared(ourGlobalToken);
 817               ourGlobalTokenUsed = false;
 818             } else {
 819               globalLock.unlock_shared();
 820             }
 821           } else if (randVal < 400) {
 822             // 40% chance of shared lock with token, 5 ways to get it
 823
 824             // delta t goes from -1 millis to 7 millis
 825             auto dt = microseconds(10 * (randVal - 100));
 826
 827             if (randVal < 400) {
 828               globalLock.lock_shared(ourGlobalToken);
 829               shared = true;
 830             } else if (randVal < 500) {
 831               shared = globalLock.try_lock_shared(ourGlobalToken);
 832             } else if (randVal < 600) {
 833               shared = globalLock.try_lock_shared_for(dt, ourGlobalToken);
 834             } else if (randVal < 800) {
 835               shared = globalLock.try_lock_shared_until(
 836                   system_clock::now() + dt, ourGlobalToken);
 837             }
 838             if (shared) {
 839               ourGlobalTokenUsed = true;
 840               ++*globalSharedCountPtr;
 841             }
 842           } else if (randVal < 800) {
 843             // 40% chance of shared lock without token
 844             auto dt = microseconds(10 * (randVal - 100));
 845             if (randVal < 400) {
 846               globalLock.lock_shared();
 847               shared = true;
 848             } else if (randVal < 500) {
 849               shared = globalLock.try_lock_shared();
 850             } else if (randVal < 600) {
 851               shared = globalLock.try_lock_shared_for(dt);
 852             } else if (randVal < 800) {
 853               shared = globalLock.try_lock_shared_until(
 854                   system_clock::now() + dt);
 855             }
 856             if (shared) {
 857               ++*globalSharedCountPtr;
 858             }
 859           } else if (randVal < 900) {
 860             // 10% change of upgrade lock
 861             globalLock.lock_upgrade();
 862             upgrade = true;
 863             ++*globalUpgradeCountPtr;
 864           } else {
 865             // 10% chance of exclusive lock, 5 ways to get it
 866
 867             // delta t goes from -1 millis to 9 millis
 868             auto dt = microseconds(100 * (randVal - 910));
 869
 870             if (randVal < 400) {
 871               globalLock.lock();
 872               exclusive = true;
 873             } else if (randVal < 500) {
 874               exclusive = globalLock.try_lock();
 875             } else if (randVal < 600) {
 876               exclusive = globalLock.try_lock_for(dt);
 877             } else if (randVal < 700) {
 878               exclusive = globalLock.try_lock_until(steady_clock::now() + dt);
 879             } else {
 880               exclusive = globalLock.try_lock_until(system_clock::now() + dt);
 881             }
 882             if (exclusive) {
 883               ++*globalExclusiveCountPtr;
 884             }
 885           }
 886         }
 887
 888         if (exclusive) {
 889           --*globalExclusiveCountPtr;
 890           globalLock.unlock();
 891         }
 892         if (upgrade) {
 893           --*globalUpgradeCountPtr;
 894           globalLock.unlock_upgrade();
 895         }
 896         if (shared) {
 897           --*globalSharedCountPtr;
 898           if (ourGlobalTokenUsed) {
 899             globalLock.unlock_shared(ourGlobalToken);
 900             ourGlobalTokenUsed = false;
 901           } else {
 902             globalLock.unlock_shared();
 903           }
 904         }
 905         for (auto& token : privateTokens) {
 906           privateLock.unlock_shared(token);
 907         }
 908       });
 909     }
 910   }
 911
 912   go.store(true);
 913   for (auto& thr : threads) {
 914     DSched::join(thr);
 915   }
 916 }
 917
 918 TEST(SharedMutex, deterministic_concurrent_readers_of_one_lock_read_prio) {
 919   for (int pass = 0; pass < 3; ++pass) {
 920     DSched sched(DSched::uniform(pass));
 921     runContendedReaders<DeterministicAtomic,
 922                         DSharedMutexReadPriority,
 923                         Locker>(1000, 3, false);
 924   }
 925 }
 926
 927 TEST(SharedMutex, deterministic_concurrent_readers_of_one_lock_write_prio) {
 928   for (int pass = 0; pass < 3; ++pass) {
 929     DSched sched(DSched::uniform(pass));
 930     runContendedReaders<DeterministicAtomic,
 931                         DSharedMutexWritePriority,
 932                         Locker>(1000, 3, false);
 933   }
 934 }
 935
 936 TEST(SharedMutex, concurrent_readers_of_one_lock_read_prio) {
 937   for (int pass = 0; pass < 10; ++pass) {
 938     runContendedReaders<atomic, SharedMutexReadPriority, Locker>(
 939         100000, 32, false);
 940   }
 941 }
 942
 943 TEST(SharedMutex, concurrent_readers_of_one_lock_write_prio) {
 944   for (int pass = 0; pass < 10; ++pass) {
 945     runContendedReaders<atomic, SharedMutexWritePriority, Locker>(
 946         100000, 32, false);
 947   }
 948 }
 949
 950 TEST(SharedMutex, deterministic_readers_of_concurrent_locks_read_prio) {
 951   for (int pass = 0; pass < 3; ++pass) {
 952     DSched sched(DSched::uniform(pass));
 953     runContendedReaders<DeterministicAtomic,
 954                         DSharedMutexReadPriority,
 955                         Locker>(1000, 3, true);
 956   }
 957 }
 958
 959 TEST(SharedMutex, deterministic_readers_of_concurrent_locks_write_prio) {
 960   for (int pass = 0; pass < 3; ++pass) {
 961     DSched sched(DSched::uniform(pass));
 962     runContendedReaders<DeterministicAtomic,
 963                         DSharedMutexWritePriority,
 964                         Locker>(1000, 3, true);
 965   }
 966 }
 967
 968 TEST(SharedMutex, readers_of_concurrent_locks_read_prio) {
 969   for (int pass = 0; pass < 10; ++pass) {
 970     runContendedReaders<atomic, SharedMutexReadPriority, TokenLocker>(
 971         100000, 32, true);
 972   }
 973 }
 974
 975 TEST(SharedMutex, readers_of_concurrent_locks_write_prio) {
 976   for (int pass = 0; pass < 10; ++pass) {
 977     runContendedReaders<atomic, SharedMutexWritePriority, TokenLocker>(
 978         100000, 32, true);
 979   }
 980 }
 981
 982 TEST(SharedMutex, deterministic_mixed_mostly_read_read_prio) {
 983   for (int pass = 0; pass < 3; ++pass) {
 984     DSched sched(DSched::uniform(pass));
 985     runMixed<DeterministicAtomic, DSharedMutexReadPriority, Locker>(
 986         1000, 3, 0.1, false);
 987   }
 988 }
 989
 990 TEST(SharedMutex, deterministic_mixed_mostly_read_write_prio) {
 991   for (int pass = 0; pass < 3; ++pass) {
 992     DSched sched(DSched::uniform(pass));
 993     runMixed<DeterministicAtomic, DSharedMutexWritePriority, Locker>(
 994         1000, 3, 0.1, false);
 995   }
 996 }
 997
 998 TEST(SharedMutex, mixed_mostly_read_read_prio) {
 999   for (int pass = 0; pass < 5; ++pass) {
1000     runMixed<atomic, SharedMutexReadPriority, TokenLocker>(
1001         50000, 32, 0.1, false);
1002   }
1003 }
1004
1005 TEST(SharedMutex, mixed_mostly_read_write_prio) {
1006   for (int pass = 0; pass < 5; ++pass) {
1007     runMixed<atomic, SharedMutexWritePriority, TokenLocker>(
1008         50000, 32, 0.1, false);
1009   }
1010 }
1011
1012 TEST(SharedMutex, deterministic_mixed_mostly_write_read_prio) {
1013   for (int pass = 0; pass < 1; ++pass) {
1014     DSched sched(DSched::uniform(pass));
1015     runMixed<DeterministicAtomic, DSharedMutexReadPriority, TokenLocker>(
1016         1000, 10, 0.9, false);
1017   }
1018 }
1019
1020 TEST(SharedMutex, deterministic_mixed_mostly_write_write_prio) {
1021   for (int pass = 0; pass < 1; ++pass) {
1022     DSched sched(DSched::uniform(pass));
1023     runMixed<DeterministicAtomic, DSharedMutexWritePriority, TokenLocker>(
1024         1000, 10, 0.9, false);
1025   }
1026 }
1027
1028 TEST(SharedMutex, deterministic_lost_wakeup_write_prio) {
1029   for (int pass = 0; pass < 10; ++pass) {
1030     DSched sched(DSched::uniformSubset(pass, 2, 200));
1031     runMixed<DeterministicAtomic, DSharedMutexWritePriority, TokenLocker>(
1032         1000, 3, 1.0, false);
1033   }
1034 }
1035
1036 TEST(SharedMutex, mixed_mostly_write_read_prio) {
1037   for (int pass = 0; pass < 5; ++pass) {
1038     runMixed<atomic, SharedMutexReadPriority, TokenLocker>(
1039         50000, 300, 0.9, false);
1040   }
1041 }
1042
1043 TEST(SharedMutex, mixed_mostly_write_write_prio) {
1044   for (int pass = 0; pass < 5; ++pass) {
1045     runMixed<atomic, SharedMutexWritePriority, TokenLocker>(
1046         50000, 300, 0.9, false);
1047   }
1048 }
1049
1050 TEST(SharedMutex, deterministic_all_ops_read_prio) {
1051   for (int pass = 0; pass < 5; ++pass) {
1052     DSched sched(DSched::uniform(pass));
1053     runAllAndValidate<DSharedMutexReadPriority, DeterministicAtomic>(1000, 8);
1054   }
1055 }
1056
1057 TEST(SharedMutex, deterministic_all_ops_write_prio) {
1058   for (int pass = 0; pass < 5; ++pass) {
1059     DSched sched(DSched::uniform(pass));
1060     runAllAndValidate<DSharedMutexWritePriority, DeterministicAtomic>(1000, 8);
1061   }
1062 }
1063
1064 TEST(SharedMutex, all_ops_read_prio) {
1065   for (int pass = 0; pass < 5; ++pass) {
1066     runAllAndValidate<SharedMutexReadPriority, atomic>(100000, 32);
1067   }
1068 }
1069
1070 TEST(SharedMutex, all_ops_write_prio) {
1071   for (int pass = 0; pass < 5; ++pass) {
1072     runAllAndValidate<SharedMutexWritePriority, atomic>(100000, 32);
1073   }
1074 }
1075
1076 FOLLY_ASSUME_FBVECTOR_COMPATIBLE(
1077     boost::optional<boost::optional<SharedMutexToken>>)
1078
1079 // Setup is a set of threads that either grab a shared lock, or exclusive
1080 // and then downgrade it, or upgrade then upgrade and downgrade, then
1081 // enqueue the shared lock to a second set of threads that just performs
1082 // unlocks.  Half of the shared locks use tokens, the others don't.
1083 template <typename Lock, template <typename> class Atom>
1084 static void runRemoteUnlock(size_t numOps,
1085                             double preWriteFraction,
1086                             double preUpgradeFraction,
1087                             size_t numSendingThreads,
1088                             size_t numReceivingThreads) {
1089   Lock globalLock;
1090   MPMCQueue<boost::optional<boost::optional<SharedMutexToken>>, Atom>
1091     queue(10);
1092   auto queuePtr = &queue; // workaround for clang crash
1093
1094   Atom<bool> go(false);
1095   auto goPtr = &go; // workaround for clang crash
1096   Atom<int> pendingSenders(numSendingThreads);
1097   auto pendingSendersPtr = &pendingSenders; // workaround for clang crash
1098   vector<thread> threads(numSendingThreads + numReceivingThreads);
1099
1100   BENCHMARK_SUSPEND {
1101     for (size_t t = 0; t < threads.size(); ++t) {
1102       threads[t] = DSched::thread([&, t, numSendingThreads] {
1103         if (t >= numSendingThreads) {
1104           // we're a receiver
1105           typename decltype(queue)::value_type elem;
1106           while (true) {
1107             queuePtr->blockingRead(elem);
1108             if (!elem) {
1109               // EOF, pass the EOF token
1110               queuePtr->blockingWrite(std::move(elem));
1111               break;
1112             }
1113             if (*elem) {
1114               globalLock.unlock_shared(**elem);
1115             } else {
1116               globalLock.unlock_shared();
1117             }
1118           }
1119           return;
1120         }
1121         // else we're a sender
1122
1123         struct drand48_data buffer;
1124         srand48_r(t, &buffer);
1125
1126         while (!goPtr->load()) {
1127           this_thread::yield();
1128         }
1129         for (size_t op = t; op < numOps; op += numSendingThreads) {
1130           long unscaledRandVal;
1131           lrand48_r(&buffer, &unscaledRandVal);
1132
1133           // randVal in [0,1]
1134           double randVal = ((double)unscaledRandVal) / 0x7fffffff;
1135
1136           // extract a bit and rescale
1137           bool useToken = randVal >= 0.5;
1138           randVal = (randVal - (useToken ? 0.5 : 0.0)) * 2;
1139
1140           boost::optional<SharedMutexToken> maybeToken;
1141
1142           if (useToken) {
1143             SharedMutexToken token;
1144             if (randVal < preWriteFraction) {
1145               globalLock.lock();
1146               globalLock.unlock_and_lock_shared(token);
1147             } else if (randVal < preWriteFraction + preUpgradeFraction / 2) {
1148               globalLock.lock_upgrade();
1149               globalLock.unlock_upgrade_and_lock_shared(token);
1150             } else if (randVal < preWriteFraction + preUpgradeFraction) {
1151               globalLock.lock_upgrade();
1152               globalLock.unlock_upgrade_and_lock();
1153               globalLock.unlock_and_lock_shared(token);
1154             } else {
1155               globalLock.lock_shared(token);
1156             }
1157             maybeToken = token;
1158           } else {
1159             if (randVal < preWriteFraction) {
1160               globalLock.lock();
1161               globalLock.unlock_and_lock_shared();
1162             } else if (randVal < preWriteFraction + preUpgradeFraction / 2) {
1163               globalLock.lock_upgrade();
1164               globalLock.unlock_upgrade_and_lock_shared();
1165             } else if (randVal < preWriteFraction + preUpgradeFraction) {
1166               globalLock.lock_upgrade();
1167               globalLock.unlock_upgrade_and_lock();
1168               globalLock.unlock_and_lock_shared();
1169             } else {
1170               globalLock.lock_shared();
1171             }
1172           }
1173
1174           // blockingWrite is emplace-like, so this automatically adds
1175           // another level of wrapping
1176           queuePtr->blockingWrite(maybeToken);
1177         }
1178         if (--*pendingSendersPtr == 0) {
1179           queuePtr->blockingWrite(boost::none);
1180         }
1181       });
1182     }
1183   }
1184
1185   go.store(true);
1186   for (auto& thr : threads) {
1187     DSched::join(thr);
1188   }
1189 }
1190
1191 TEST(SharedMutex, deterministic_remote_write_prio) {
1192   for (int pass = 0; pass < 1; ++pass) {
1193     DSched sched(DSched::uniform(pass));
1194     runRemoteUnlock<DSharedMutexWritePriority, DeterministicAtomic>(
1195         500, 0.1, 0.1, 5, 5);
1196   }
1197 }
1198
1199 TEST(SharedMutex, deterministic_remote_read_prio) {
1200   for (int pass = 0; pass < 1; ++pass) {
1201     DSched sched(DSched::uniform(pass));
1202     runRemoteUnlock<DSharedMutexReadPriority, DeterministicAtomic>(
1203         500, 0.1, 0.1, 5, 5);
1204   }
1205 }
1206
1207 TEST(SharedMutex, remote_write_prio) {
1208   for (int pass = 0; pass < 10; ++pass) {
1209     runRemoteUnlock<SharedMutexWritePriority, atomic>(100000, 0.1, 0.1, 5, 5);
1210   }
1211 }
1212
1213 TEST(SharedMutex, remote_read_prio) {
1214   for (int pass = 0; pass < 100; ++pass) {
1215     runRemoteUnlock<SharedMutexReadPriority, atomic>(100000, 0.1, 0.1, 5, 5);
1216   }
1217 }
1218
1219 static void burn(size_t n) {
1220   for (size_t i = 0; i < n; ++i) {
1221     folly::doNotOptimizeAway(i);
1222   }
1223 }
1224
1225 // Two threads and three locks, arranged so that they have to proceed
1226 // in turn with reader/writer conflict
1227 template <typename Lock, template <typename> class Atom = atomic>
1228 static void runPingPong(size_t numRounds, size_t burnCount) {
1229   char padding1[56];
1230   pair<Lock, char[56]> locks[3];
1231   char padding2[56];
1232
1233   Atom<int> avail(0);
1234   auto availPtr = &avail; // workaround for clang crash
1235   Atom<bool> go(false);
1236   auto goPtr = &go; // workaround for clang crash
1237   vector<thread> threads(2);
1238
1239   locks[0].first.lock();
1240   locks[1].first.lock();
1241   locks[2].first.lock_shared();
1242
1243   BENCHMARK_SUSPEND {
1244     threads[0] = DSched::thread([&] {
1245       ++*availPtr;
1246       while (!goPtr->load()) {
1247         this_thread::yield();
1248       }
1249       for (size_t i = 0; i < numRounds; ++i) {
1250         locks[i % 3].first.unlock();
1251         locks[(i + 2) % 3].first.lock();
1252         burn(burnCount);
1253       }
1254     });
1255     threads[1] = DSched::thread([&] {
1256       ++*availPtr;
1257       while (!goPtr->load()) {
1258         this_thread::yield();
1259       }
1260       for (size_t i = 0; i < numRounds; ++i) {
1261         locks[i % 3].first.lock_shared();
1262         burn(burnCount);
1263         locks[(i + 2) % 3].first.unlock_shared();
1264       }
1265     });
1266
1267     while (avail.load() < 2) {
1268       this_thread::yield();
1269     }
1270   }
1271
1272   go.store(true);
1273   for (auto& thr : threads) {
1274     DSched::join(thr);
1275   }
1276   locks[numRounds % 3].first.unlock();
1277   locks[(numRounds + 1) % 3].first.unlock();
1278   locks[(numRounds + 2) % 3].first.unlock_shared();
1279 }
1280
1281 static void folly_rwspin_ping_pong(size_t n, size_t scale, size_t burnCount) {
1282   runPingPong<RWSpinLock>(n / scale, burnCount);
1283 }
1284
1285 static void shmtx_w_bare_ping_pong(size_t n, size_t scale, size_t burnCount) {
1286   runPingPong<SharedMutexWritePriority>(n / scale, burnCount);
1287 }
1288
1289 static void shmtx_r_bare_ping_pong(size_t n, size_t scale, size_t burnCount) {
1290   runPingPong<SharedMutexReadPriority>(n / scale, burnCount);
1291 }
1292
1293 static void folly_ticket_ping_pong(size_t n, size_t scale, size_t burnCount) {
1294   runPingPong<RWTicketSpinLock64>(n / scale, burnCount);
1295 }
1296
1297 static void boost_shared_ping_pong(size_t n, size_t scale, size_t burnCount) {
1298   runPingPong<boost::shared_mutex>(n / scale, burnCount);
1299 }
1300
1301 static void pthrd_rwlock_ping_pong(size_t n, size_t scale, size_t burnCount) {
1302   runPingPong<PosixRWLock>(n / scale, burnCount);
1303 }
1304
1305 TEST(SharedMutex, deterministic_ping_pong_write_prio) {
1306   for (int pass = 0; pass < 1; ++pass) {
1307     DSched sched(DSched::uniform(pass));
1308     runPingPong<DSharedMutexWritePriority, DeterministicAtomic>(500, 0);
1309   }
1310 }
1311
1312 TEST(SharedMutex, deterministic_ping_pong_read_prio) {
1313   for (int pass = 0; pass < 1; ++pass) {
1314     DSched sched(DSched::uniform(pass));
1315     runPingPong<DSharedMutexReadPriority, DeterministicAtomic>(500, 0);
1316   }
1317 }
1318
1319 TEST(SharedMutex, ping_pong_write_prio) {
1320   for (int pass = 0; pass < 1; ++pass) {
1321     runPingPong<SharedMutexWritePriority, atomic>(50000, 0);
1322   }
1323 }
1324
1325 TEST(SharedMutex, ping_pong_read_prio) {
1326   for (int pass = 0; pass < 1; ++pass) {
1327     runPingPong<SharedMutexReadPriority, atomic>(50000, 0);
1328   }
1329 }
1330
1331 // This is here so you can tell how much of the runtime reported by the
1332 // more complex harnesses is due to the harness, although due to the
1333 // magic of compiler optimization it may also be slower
1334 BENCHMARK(single_thread_lock_shared_unlock_shared, iters) {
1335   SharedMutex lock;
1336   for (size_t n = 0; n < iters; ++n) {
1337     SharedMutex::Token token;
1338     lock.lock_shared(token);
1339     folly::doNotOptimizeAway(0);
1340     lock.unlock_shared(token);
1341   }
1342 }
1343
1344 BENCHMARK(single_thread_lock_unlock, iters) {
1345   SharedMutex lock;
1346   for (size_t n = 0; n < iters; ++n) {
1347     lock.lock();
1348     folly::doNotOptimizeAway(0);
1349     lock.unlock();
1350   }
1351 }
1352
1353 #define BENCH_BASE(args...) BENCHMARK_NAMED_PARAM(args)
1354 #define BENCH_REL(args...) BENCHMARK_RELATIVE_NAMED_PARAM(args)
1355
1356 // 100% reads.  Best-case scenario for deferred locks.  Lock is colocated
1357 // with read data, so inline lock takes cache miss every time but deferred
1358 // lock has only cache hits and local access.
1359 BENCHMARK_DRAW_LINE()
1360 BENCHMARK_DRAW_LINE()
1361 BENCH_BASE(folly_rwspin_reads, 1thread, 1, false)
1362 BENCH_REL (shmtx_wr_pri_reads, 1thread, 1, false)
1363 BENCH_REL (shmtx_w_bare_reads, 1thread, 1, false)
1364 BENCH_REL (shmtx_rd_pri_reads, 1thread, 1, false)
1365 BENCH_REL (shmtx_r_bare_reads, 1thread, 1, false)
1366 BENCH_REL (folly_ticket_reads, 1thread, 1, false)
1367 BENCH_REL (boost_shared_reads, 1thread, 1, false)
1368 BENCH_REL (pthrd_rwlock_reads, 1thread, 1, false)
1369 BENCHMARK_DRAW_LINE()
1370 BENCH_BASE(folly_rwspin_reads, 2thread, 2, false)
1371 BENCH_REL (shmtx_wr_pri_reads, 2thread, 2, false)
1372 BENCH_REL (shmtx_w_bare_reads, 2thread, 2, false)
1373 BENCH_REL (shmtx_rd_pri_reads, 2thread, 2, false)
1374 BENCH_REL (shmtx_r_bare_reads, 2thread, 2, false)
1375 BENCH_REL (folly_ticket_reads, 2thread, 2, false)
1376 BENCH_REL (boost_shared_reads, 2thread, 2, false)
1377 BENCH_REL (pthrd_rwlock_reads, 2thread, 2, false)
1378 BENCHMARK_DRAW_LINE()
1379 BENCH_BASE(folly_rwspin_reads, 4thread, 4, false)
1380 BENCH_REL (shmtx_wr_pri_reads, 4thread, 4, false)
1381 BENCH_REL (shmtx_w_bare_reads, 4thread, 4, false)
1382 BENCH_REL (shmtx_rd_pri_reads, 4thread, 4, false)
1383 BENCH_REL (shmtx_r_bare_reads, 4thread, 4, false)
1384 BENCH_REL (folly_ticket_reads, 4thread, 4, false)
1385 BENCH_REL (boost_shared_reads, 4thread, 4, false)
1386 BENCH_REL (pthrd_rwlock_reads, 4thread, 4, false)
1387 BENCHMARK_DRAW_LINE()
1388 BENCH_BASE(folly_rwspin_reads, 8thread, 8, false)
1389 BENCH_REL (shmtx_wr_pri_reads, 8thread, 8, false)
1390 BENCH_REL (shmtx_w_bare_reads, 8thread, 8, false)
1391 BENCH_REL (shmtx_rd_pri_reads, 8thread, 8, false)
1392 BENCH_REL (shmtx_r_bare_reads, 8thread, 8, false)
1393 BENCH_REL (folly_ticket_reads, 8thread, 8, false)
1394 BENCH_REL (boost_shared_reads, 8thread, 8, false)
1395 BENCH_REL (pthrd_rwlock_reads, 8thread, 8, false)
1396 BENCHMARK_DRAW_LINE()
1397 BENCH_BASE(folly_rwspin_reads, 16thread, 16, false)
1398 BENCH_REL (shmtx_wr_pri_reads, 16thread, 16, false)
1399 BENCH_REL (shmtx_w_bare_reads, 16thread, 16, false)
1400 BENCH_REL (shmtx_rd_pri_reads, 16thread, 16, false)
1401 BENCH_REL (shmtx_r_bare_reads, 16thread, 16, false)
1402 BENCH_REL (folly_ticket_reads, 16thread, 16, false)
1403 BENCH_REL (boost_shared_reads, 16thread, 16, false)
1404 BENCH_REL (pthrd_rwlock_reads, 16thread, 16, false)
1405 BENCHMARK_DRAW_LINE()
1406 BENCH_BASE(folly_rwspin_reads, 32thread, 32, false)
1407 BENCH_REL (shmtx_wr_pri_reads, 32thread, 32, false)
1408 BENCH_REL (shmtx_w_bare_reads, 32thread, 32, false)
1409 BENCH_REL (shmtx_rd_pri_reads, 32thread, 32, false)
1410 BENCH_REL (shmtx_r_bare_reads, 32thread, 32, false)
1411 BENCH_REL (folly_ticket_reads, 32thread, 32, false)
1412 BENCH_REL (boost_shared_reads, 32thread, 32, false)
1413 BENCH_REL (pthrd_rwlock_reads, 32thread, 32, false)
1414 BENCHMARK_DRAW_LINE()
1415 BENCH_BASE(folly_rwspin_reads, 64thread, 64, false)
1416 BENCH_REL (shmtx_wr_pri_reads, 64thread, 64, false)
1417 BENCH_REL (shmtx_w_bare_reads, 64thread, 64, false)
1418 BENCH_REL (shmtx_rd_pri_reads, 64thread, 64, false)
1419 BENCH_REL (shmtx_r_bare_reads, 64thread, 64, false)
1420 BENCH_REL (folly_ticket_reads, 64thread, 64, false)
1421 BENCH_REL (boost_shared_reads, 64thread, 64, false)
1422 BENCH_REL (pthrd_rwlock_reads, 64thread, 64, false)
1423
1424 // 1 lock used by everybody, 100% writes.  Threads only hurt, but it is
1425 // good to not fail catastrophically.  Compare to single_thread_lock_unlock
1426 // to see the overhead of the generic driver (and its pseudo-random number
1427 // generator).  pthrd_mutex_ is a pthread_mutex_t (default, not adaptive),
1428 // which is better than any of the reader-writer locks for this scenario.
1429 BENCHMARK_DRAW_LINE()
1430 BENCHMARK_DRAW_LINE()
1431 BENCH_BASE(folly_rwspin, 1thread_all_write, 1, 1.0, false)
1432 BENCH_REL (shmtx_wr_pri, 1thread_all_write, 1, 1.0, false)
1433 BENCH_REL (shmtx_rd_pri, 1thread_all_write, 1, 1.0, false)
1434 BENCH_REL (folly_ticket, 1thread_all_write, 1, 1.0, false)
1435 BENCH_REL (boost_shared, 1thread_all_write, 1, 1.0, false)
1436 BENCH_REL (pthrd_rwlock, 1thread_all_write, 1, 1.0, false)
1437 BENCH_REL (pthrd_mutex_, 1thread_all_write, 1, 1.0, false)
1438 BENCHMARK_DRAW_LINE()
1439 BENCH_BASE(folly_rwspin, 2thread_all_write, 2, 1.0, false)
1440 BENCH_REL (shmtx_wr_pri, 2thread_all_write, 2, 1.0, false)
1441 BENCH_REL (shmtx_rd_pri, 2thread_all_write, 2, 1.0, false)
1442 BENCH_REL (folly_ticket, 2thread_all_write, 2, 1.0, false)
1443 BENCH_REL (boost_shared, 2thread_all_write, 2, 1.0, false)
1444 BENCH_REL (pthrd_rwlock, 2thread_all_write, 2, 1.0, false)
1445 BENCH_REL (pthrd_mutex_, 2thread_all_write, 2, 1.0, false)
1446 BENCHMARK_DRAW_LINE()
1447 BENCH_BASE(folly_rwspin, 4thread_all_write, 4, 1.0, false)
1448 BENCH_REL (shmtx_wr_pri, 4thread_all_write, 4, 1.0, false)
1449 BENCH_REL (shmtx_rd_pri, 4thread_all_write, 4, 1.0, false)
1450 BENCH_REL (folly_ticket, 4thread_all_write, 4, 1.0, false)
1451 BENCH_REL (boost_shared, 4thread_all_write, 4, 1.0, false)
1452 BENCH_REL (pthrd_rwlock, 4thread_all_write, 4, 1.0, false)
1453 BENCH_REL (pthrd_mutex_, 4thread_all_write, 4, 1.0, false)
1454 BENCHMARK_DRAW_LINE()
1455 BENCH_BASE(folly_rwspin, 8thread_all_write, 8, 1.0, false)
1456 BENCH_REL (shmtx_wr_pri, 8thread_all_write, 8, 1.0, false)
1457 BENCH_REL (shmtx_rd_pri, 8thread_all_write, 8, 1.0, false)
1458 BENCH_REL (folly_ticket, 8thread_all_write, 8, 1.0, false)
1459 BENCH_REL (boost_shared, 8thread_all_write, 8, 1.0, false)
1460 BENCH_REL (pthrd_rwlock, 8thread_all_write, 8, 1.0, false)
1461 BENCH_REL (pthrd_mutex_, 8thread_all_write, 8, 1.0, false)
1462 BENCHMARK_DRAW_LINE()
1463 BENCH_BASE(folly_rwspin, 16thread_all_write, 16, 1.0, false)
1464 BENCH_REL (shmtx_wr_pri, 16thread_all_write, 16, 1.0, false)
1465 BENCH_REL (shmtx_rd_pri, 16thread_all_write, 16, 1.0, false)
1466 BENCH_REL (folly_ticket, 16thread_all_write, 16, 1.0, false)
1467 BENCH_REL (boost_shared, 16thread_all_write, 16, 1.0, false)
1468 BENCH_REL (pthrd_rwlock, 16thread_all_write, 16, 1.0, false)
1469 BENCH_REL (pthrd_mutex_, 16thread_all_write, 16, 1.0, false)
1470 BENCHMARK_DRAW_LINE()
1471 BENCH_BASE(folly_rwspin, 32thread_all_write, 32, 1.0, false)
1472 BENCH_REL (shmtx_wr_pri, 32thread_all_write, 32, 1.0, false)
1473 BENCH_REL (shmtx_rd_pri, 32thread_all_write, 32, 1.0, false)
1474 BENCH_REL (folly_ticket, 32thread_all_write, 32, 1.0, false)
1475 BENCH_REL (boost_shared, 32thread_all_write, 32, 1.0, false)
1476 BENCH_REL (pthrd_rwlock, 32thread_all_write, 32, 1.0, false)
1477 BENCH_REL (pthrd_mutex_, 32thread_all_write, 32, 1.0, false)
1478 BENCHMARK_DRAW_LINE()
1479 BENCH_BASE(folly_rwspin, 64thread_all_write, 64, 1.0, false)
1480 BENCH_REL (shmtx_wr_pri, 64thread_all_write, 64, 1.0, false)
1481 BENCH_REL (shmtx_rd_pri, 64thread_all_write, 64, 1.0, false)
1482 BENCH_REL (folly_ticket, 64thread_all_write, 64, 1.0, false)
1483 BENCH_REL (boost_shared, 64thread_all_write, 64, 1.0, false)
1484 BENCH_REL (pthrd_rwlock, 64thread_all_write, 64, 1.0, false)
1485 BENCH_REL (pthrd_mutex_, 64thread_all_write, 64, 1.0, false)
1486
1487 // 1 lock used by everybody, 10% writes.  Not much scaling to be had.  Perf
1488 // is best at 1 thread, once you've got multiple threads > 8 threads hurts.
1489 BENCHMARK_DRAW_LINE()
1490 BENCHMARK_DRAW_LINE()
1491 BENCH_BASE(folly_rwspin, 1thread_10pct_write, 1, 0.10, false)
1492 BENCH_REL (shmtx_wr_pri, 1thread_10pct_write, 1, 0.10, false)
1493 BENCH_REL (shmtx_rd_pri, 1thread_10pct_write, 1, 0.10, false)
1494 BENCH_REL (folly_ticket, 1thread_10pct_write, 1, 0.10, false)
1495 BENCH_REL (boost_shared, 1thread_10pct_write, 1, 0.10, false)
1496 BENCH_REL (pthrd_rwlock, 1thread_10pct_write, 1, 0.10, false)
1497 BENCHMARK_DRAW_LINE()
1498 BENCH_BASE(folly_rwspin, 2thread_10pct_write, 2, 0.10, false)
1499 BENCH_REL (shmtx_wr_pri, 2thread_10pct_write, 2, 0.10, false)
1500 BENCH_REL (shmtx_rd_pri, 2thread_10pct_write, 2, 0.10, false)
1501 BENCH_REL (folly_ticket, 2thread_10pct_write, 2, 0.10, false)
1502 BENCH_REL (boost_shared, 2thread_10pct_write, 2, 0.10, false)
1503 BENCH_REL (pthrd_rwlock, 2thread_10pct_write, 2, 0.10, false)
1504 BENCHMARK_DRAW_LINE()
1505 BENCH_BASE(folly_rwspin, 4thread_10pct_write, 4, 0.10, false)
1506 BENCH_REL (shmtx_wr_pri, 4thread_10pct_write, 4, 0.10, false)
1507 BENCH_REL (shmtx_rd_pri, 4thread_10pct_write, 4, 0.10, false)
1508 BENCH_REL (folly_ticket, 4thread_10pct_write, 4, 0.10, false)
1509 BENCH_REL (boost_shared, 4thread_10pct_write, 4, 0.10, false)
1510 BENCH_REL (pthrd_rwlock, 4thread_10pct_write, 4, 0.10, false)
1511 BENCHMARK_DRAW_LINE()
1512 BENCH_BASE(folly_rwspin, 8thread_10pct_write, 8, 0.10, false)
1513 BENCH_REL (shmtx_wr_pri, 8thread_10pct_write, 8, 0.10, false)
1514 BENCH_REL (shmtx_rd_pri, 8thread_10pct_write, 8, 0.10, false)
1515 BENCH_REL (folly_ticket, 8thread_10pct_write, 8, 0.10, false)
1516 BENCH_REL (boost_shared, 8thread_10pct_write, 8, 0.10, false)
1517 BENCH_REL (pthrd_rwlock, 8thread_10pct_write, 8, 0.10, false)
1518 BENCHMARK_DRAW_LINE()
1519 BENCH_BASE(folly_rwspin, 16thread_10pct_write, 16, 0.10, false)
1520 BENCH_REL (shmtx_wr_pri, 16thread_10pct_write, 16, 0.10, false)
1521 BENCH_REL (shmtx_rd_pri, 16thread_10pct_write, 16, 0.10, false)
1522 BENCH_REL (folly_ticket, 16thread_10pct_write, 16, 0.10, false)
1523 BENCH_REL (boost_shared, 16thread_10pct_write, 16, 0.10, false)
1524 BENCH_REL (pthrd_rwlock, 16thread_10pct_write, 16, 0.10, false)
1525 BENCHMARK_DRAW_LINE()
1526 BENCH_BASE(folly_rwspin, 32thread_10pct_write, 32, 0.10, false)
1527 BENCH_REL (shmtx_wr_pri, 32thread_10pct_write, 32, 0.10, false)
1528 BENCH_REL (shmtx_rd_pri, 32thread_10pct_write, 32, 0.10, false)
1529 BENCH_REL (folly_ticket, 32thread_10pct_write, 32, 0.10, false)
1530 BENCH_REL (boost_shared, 32thread_10pct_write, 32, 0.10, false)
1531 BENCH_REL (pthrd_rwlock, 32thread_10pct_write, 32, 0.10, false)
1532 BENCHMARK_DRAW_LINE()
1533 BENCH_BASE(folly_rwspin, 64thread_10pct_write, 64, 0.10, false)
1534 BENCH_REL (shmtx_wr_pri, 64thread_10pct_write, 64, 0.10, false)
1535 BENCH_REL (shmtx_rd_pri, 64thread_10pct_write, 64, 0.10, false)
1536 BENCH_REL (folly_ticket, 64thread_10pct_write, 64, 0.10, false)
1537 BENCH_REL (boost_shared, 64thread_10pct_write, 64, 0.10, false)
1538 BENCH_REL (pthrd_rwlock, 64thread_10pct_write, 64, 0.10, false)
1539
1540 // 1 lock used by everybody, 1% writes.  This is a more realistic example
1541 // than the concurrent_*_reads benchmark, but still shows SharedMutex locks
1542 // winning over all of the others
1543 BENCHMARK_DRAW_LINE()
1544 BENCHMARK_DRAW_LINE()
1545 BENCH_BASE(folly_rwspin, 1thread_1pct_write, 1, 0.01, false)
1546 BENCH_REL (shmtx_wr_pri, 1thread_1pct_write, 1, 0.01, false)
1547 BENCH_REL (shmtx_w_bare, 1thread_1pct_write, 1, 0.01, false)
1548 BENCH_REL (shmtx_rd_pri, 1thread_1pct_write, 1, 0.01, false)
1549 BENCH_REL (shmtx_r_bare, 1thread_1pct_write, 1, 0.01, false)
1550 BENCH_REL (folly_ticket, 1thread_1pct_write, 1, 0.01, false)
1551 BENCH_REL (boost_shared, 1thread_1pct_write, 1, 0.01, false)
1552 BENCH_REL (pthrd_rwlock, 1thread_1pct_write, 1, 0.01, false)
1553 BENCHMARK_DRAW_LINE()
1554 BENCH_BASE(folly_rwspin, 2thread_1pct_write, 2, 0.01, false)
1555 BENCH_REL (shmtx_wr_pri, 2thread_1pct_write, 2, 0.01, false)
1556 BENCH_REL (shmtx_w_bare, 2thread_1pct_write, 2, 0.01, false)
1557 BENCH_REL (shmtx_rd_pri, 2thread_1pct_write, 2, 0.01, false)
1558 BENCH_REL (shmtx_r_bare, 2thread_1pct_write, 2, 0.01, false)
1559 BENCH_REL (folly_ticket, 2thread_1pct_write, 2, 0.01, false)
1560 BENCH_REL (boost_shared, 2thread_1pct_write, 2, 0.01, false)
1561 BENCH_REL (pthrd_rwlock, 2thread_1pct_write, 2, 0.01, false)
1562 BENCHMARK_DRAW_LINE()
1563 BENCH_BASE(folly_rwspin, 4thread_1pct_write, 4, 0.01, false)
1564 BENCH_REL (shmtx_wr_pri, 4thread_1pct_write, 4, 0.01, false)
1565 BENCH_REL (shmtx_w_bare, 4thread_1pct_write, 4, 0.01, false)
1566 BENCH_REL (shmtx_rd_pri, 4thread_1pct_write, 4, 0.01, false)
1567 BENCH_REL (shmtx_r_bare, 4thread_1pct_write, 4, 0.01, false)
1568 BENCH_REL (folly_ticket, 4thread_1pct_write, 4, 0.01, false)
1569 BENCH_REL (boost_shared, 4thread_1pct_write, 4, 0.01, false)
1570 BENCH_REL (pthrd_rwlock, 4thread_1pct_write, 4, 0.01, false)
1571 BENCHMARK_DRAW_LINE()
1572 BENCH_BASE(folly_rwspin, 8thread_1pct_write, 8, 0.01, false)
1573 BENCH_REL (shmtx_wr_pri, 8thread_1pct_write, 8, 0.01, false)
1574 BENCH_REL (shmtx_w_bare, 8thread_1pct_write, 8, 0.01, false)
1575 BENCH_REL (shmtx_rd_pri, 8thread_1pct_write, 8, 0.01, false)
1576 BENCH_REL (shmtx_r_bare, 8thread_1pct_write, 8, 0.01, false)
1577 BENCH_REL (folly_ticket, 8thread_1pct_write, 8, 0.01, false)
1578 BENCH_REL (boost_shared, 8thread_1pct_write, 8, 0.01, false)
1579 BENCH_REL (pthrd_rwlock, 8thread_1pct_write, 8, 0.01, false)
1580 BENCHMARK_DRAW_LINE()
1581 BENCH_BASE(folly_rwspin, 16thread_1pct_write, 16, 0.01, false)
1582 BENCH_REL (shmtx_wr_pri, 16thread_1pct_write, 16, 0.01, false)
1583 BENCH_REL (shmtx_w_bare, 16thread_1pct_write, 16, 0.01, false)
1584 BENCH_REL (shmtx_rd_pri, 16thread_1pct_write, 16, 0.01, false)
1585 BENCH_REL (shmtx_r_bare, 16thread_1pct_write, 16, 0.01, false)
1586 BENCH_REL (folly_ticket, 16thread_1pct_write, 16, 0.01, false)
1587 BENCH_REL (boost_shared, 16thread_1pct_write, 16, 0.01, false)
1588 BENCH_REL (pthrd_rwlock, 16thread_1pct_write, 16, 0.01, false)
1589 BENCHMARK_DRAW_LINE()
1590 BENCH_BASE(folly_rwspin, 32thread_1pct_write, 32, 0.01, false)
1591 BENCH_REL (shmtx_wr_pri, 32thread_1pct_write, 32, 0.01, false)
1592 BENCH_REL (shmtx_w_bare, 32thread_1pct_write, 32, 0.01, false)
1593 BENCH_REL (shmtx_rd_pri, 32thread_1pct_write, 32, 0.01, false)
1594 BENCH_REL (shmtx_r_bare, 32thread_1pct_write, 32, 0.01, false)
1595 BENCH_REL (folly_ticket, 32thread_1pct_write, 32, 0.01, false)
1596 BENCH_REL (boost_shared, 32thread_1pct_write, 32, 0.01, false)
1597 BENCH_REL (pthrd_rwlock, 32thread_1pct_write, 32, 0.01, false)
1598 BENCHMARK_DRAW_LINE()
1599 BENCH_BASE(folly_rwspin, 64thread_1pct_write, 64, 0.01, false)
1600 BENCH_REL (shmtx_wr_pri, 64thread_1pct_write, 64, 0.01, false)
1601 BENCH_REL (shmtx_w_bare, 64thread_1pct_write, 64, 0.01, false)
1602 BENCH_REL (shmtx_rd_pri, 64thread_1pct_write, 64, 0.01, false)
1603 BENCH_REL (shmtx_r_bare, 64thread_1pct_write, 64, 0.01, false)
1604 BENCH_REL (folly_ticket, 64thread_1pct_write, 64, 0.01, false)
1605 BENCH_REL (boost_shared, 64thread_1pct_write, 64, 0.01, false)
1606 BENCH_REL (pthrd_rwlock, 64thread_1pct_write, 64, 0.01, false)
1607
1608 // Worst case scenario for deferred locks. No actual sharing, likely that
1609 // read operations will have to first set the kDeferredReadersPossibleBit,
1610 // and likely that writers will have to scan deferredReaders[].
1611 BENCHMARK_DRAW_LINE()
1612 BENCH_BASE(folly_rwspin, 2thr_2lock_50pct_write, 2, 0.50, true)
1613 BENCH_REL (shmtx_wr_pri, 2thr_2lock_50pct_write, 2, 0.50, true)
1614 BENCH_REL (shmtx_rd_pri, 2thr_2lock_50pct_write, 2, 0.50, true)
1615 BENCH_BASE(folly_rwspin, 4thr_4lock_50pct_write, 4, 0.50, true)
1616 BENCH_REL (shmtx_wr_pri, 4thr_4lock_50pct_write, 4, 0.50, true)
1617 BENCH_REL (shmtx_rd_pri, 4thr_4lock_50pct_write, 4, 0.50, true)
1618 BENCH_BASE(folly_rwspin, 8thr_8lock_50pct_write, 8, 0.50, true)
1619 BENCH_REL (shmtx_wr_pri, 8thr_8lock_50pct_write, 8, 0.50, true)
1620 BENCH_REL (shmtx_rd_pri, 8thr_8lock_50pct_write, 8, 0.50, true)
1621 BENCH_BASE(folly_rwspin, 16thr_16lock_50pct_write, 16, 0.50, true)
1622 BENCH_REL (shmtx_wr_pri, 16thr_16lock_50pct_write, 16, 0.50, true)
1623 BENCH_REL (shmtx_rd_pri, 16thr_16lock_50pct_write, 16, 0.50, true)
1624 BENCH_BASE(folly_rwspin, 32thr_32lock_50pct_write, 32, 0.50, true)
1625 BENCH_REL (shmtx_wr_pri, 32thr_32lock_50pct_write, 32, 0.50, true)
1626 BENCH_REL (shmtx_rd_pri, 32thr_32lock_50pct_write, 32, 0.50, true)
1627 BENCH_BASE(folly_rwspin, 64thr_64lock_50pct_write, 64, 0.50, true)
1628 BENCH_REL (shmtx_wr_pri, 64thr_64lock_50pct_write, 64, 0.50, true)
1629 BENCH_REL (shmtx_rd_pri, 64thr_64lock_50pct_write, 64, 0.50, true)
1630 BENCHMARK_DRAW_LINE()
1631 BENCH_BASE(folly_rwspin, 2thr_2lock_10pct_write, 2, 0.10, true)
1632 BENCH_REL (shmtx_wr_pri, 2thr_2lock_10pct_write, 2, 0.10, true)
1633 BENCH_REL (shmtx_rd_pri, 2thr_2lock_10pct_write, 2, 0.10, true)
1634 BENCH_BASE(folly_rwspin, 4thr_4lock_10pct_write, 4, 0.10, true)
1635 BENCH_REL (shmtx_wr_pri, 4thr_4lock_10pct_write, 4, 0.10, true)
1636 BENCH_REL (shmtx_rd_pri, 4thr_4lock_10pct_write, 4, 0.10, true)
1637 BENCH_BASE(folly_rwspin, 8thr_8lock_10pct_write, 8, 0.10, true)
1638 BENCH_REL (shmtx_wr_pri, 8thr_8lock_10pct_write, 8, 0.10, true)
1639 BENCH_REL (shmtx_rd_pri, 8thr_8lock_10pct_write, 8, 0.10, true)
1640 BENCH_BASE(folly_rwspin, 16thr_16lock_10pct_write, 16, 0.10, true)
1641 BENCH_REL (shmtx_wr_pri, 16thr_16lock_10pct_write, 16, 0.10, true)
1642 BENCH_REL (shmtx_rd_pri, 16thr_16lock_10pct_write, 16, 0.10, true)
1643 BENCH_BASE(folly_rwspin, 32thr_32lock_10pct_write, 32, 0.10, true)
1644 BENCH_REL (shmtx_wr_pri, 32thr_32lock_10pct_write, 32, 0.10, true)
1645 BENCH_REL (shmtx_rd_pri, 32thr_32lock_10pct_write, 32, 0.10, true)
1646 BENCH_BASE(folly_rwspin, 64thr_64lock_10pct_write, 64, 0.10, true)
1647 BENCH_REL (shmtx_wr_pri, 64thr_64lock_10pct_write, 64, 0.10, true)
1648 BENCH_REL (shmtx_rd_pri, 64thr_64lock_10pct_write, 64, 0.10, true)
1649 BENCHMARK_DRAW_LINE()
1650 BENCH_BASE(folly_rwspin, 2thr_2lock_1pct_write, 2, 0.01, true)
1651 BENCH_REL (shmtx_wr_pri, 2thr_2lock_1pct_write, 2, 0.01, true)
1652 BENCH_REL (shmtx_rd_pri, 2thr_2lock_1pct_write, 2, 0.01, true)
1653 BENCH_BASE(folly_rwspin, 4thr_4lock_1pct_write, 4, 0.01, true)
1654 BENCH_REL (shmtx_wr_pri, 4thr_4lock_1pct_write, 4, 0.01, true)
1655 BENCH_REL (shmtx_rd_pri, 4thr_4lock_1pct_write, 4, 0.01, true)
1656 BENCH_BASE(folly_rwspin, 8thr_8lock_1pct_write, 8, 0.01, true)
1657 BENCH_REL (shmtx_wr_pri, 8thr_8lock_1pct_write, 8, 0.01, true)
1658 BENCH_REL (shmtx_rd_pri, 8thr_8lock_1pct_write, 8, 0.01, true)
1659 BENCH_BASE(folly_rwspin, 16thr_16lock_1pct_write, 16, 0.01, true)
1660 BENCH_REL (shmtx_wr_pri, 16thr_16lock_1pct_write, 16, 0.01, true)
1661 BENCH_REL (shmtx_rd_pri, 16thr_16lock_1pct_write, 16, 0.01, true)
1662 BENCH_BASE(folly_rwspin, 32thr_32lock_1pct_write, 32, 0.01, true)
1663 BENCH_REL (shmtx_wr_pri, 32thr_32lock_1pct_write, 32, 0.01, true)
1664 BENCH_REL (shmtx_rd_pri, 32thr_32lock_1pct_write, 32, 0.01, true)
1665 BENCH_BASE(folly_rwspin, 64thr_64lock_1pct_write, 64, 0.01, true)
1666 BENCH_REL (shmtx_wr_pri, 64thr_64lock_1pct_write, 64, 0.01, true)
1667 BENCH_REL (shmtx_rd_pri, 64thr_64lock_1pct_write, 64, 0.01, true)
1668
1669 // Ping-pong tests have a scaled number of iterations, because their burn
1670 // loop would make them too slow otherwise.  Ping-pong with burn count of
1671 // 100k or 300k shows the advantage of soft-spin, reducing the cost of
1672 // each wakeup by about 20 usec.  (Take benchmark reported difference,
1673 // ~400 nanos, multiply by the scale of 100, then divide by 2 because
1674 // each round has two wakeups.)
1675 BENCHMARK_DRAW_LINE()
1676 BENCHMARK_DRAW_LINE()
1677 BENCH_BASE(folly_rwspin_ping_pong, burn0, 1, 0)
1678 BENCH_REL (shmtx_w_bare_ping_pong, burn0, 1, 0)
1679 BENCH_REL (shmtx_r_bare_ping_pong, burn0, 1, 0)
1680 BENCH_REL (folly_ticket_ping_pong, burn0, 1, 0)
1681 BENCH_REL (boost_shared_ping_pong, burn0, 1, 0)
1682 BENCH_REL (pthrd_rwlock_ping_pong, burn0, 1, 0)
1683 BENCHMARK_DRAW_LINE()
1684 BENCH_BASE(folly_rwspin_ping_pong, burn100k, 100, 100000)
1685 BENCH_REL (shmtx_w_bare_ping_pong, burn100k, 100, 100000)
1686 BENCH_REL (shmtx_r_bare_ping_pong, burn100k, 100, 100000)
1687 BENCH_REL (folly_ticket_ping_pong, burn100k, 100, 100000)
1688 BENCH_REL (boost_shared_ping_pong, burn100k, 100, 100000)
1689 BENCH_REL (pthrd_rwlock_ping_pong, burn100k, 100, 100000)
1690 BENCHMARK_DRAW_LINE()
1691 BENCH_BASE(folly_rwspin_ping_pong, burn300k, 100, 300000)
1692 BENCH_REL (shmtx_w_bare_ping_pong, burn300k, 100, 300000)
1693 BENCH_REL (shmtx_r_bare_ping_pong, burn300k, 100, 300000)
1694 BENCH_REL (folly_ticket_ping_pong, burn300k, 100, 300000)
1695 BENCH_REL (boost_shared_ping_pong, burn300k, 100, 300000)
1696 BENCH_REL (pthrd_rwlock_ping_pong, burn300k, 100, 300000)
1697 BENCHMARK_DRAW_LINE()
1698 BENCH_BASE(folly_rwspin_ping_pong, burn1M, 1000, 1000000)
1699 BENCH_REL (shmtx_w_bare_ping_pong, burn1M, 1000, 1000000)
1700 BENCH_REL (shmtx_r_bare_ping_pong, burn1M, 1000, 1000000)
1701 BENCH_REL (folly_ticket_ping_pong, burn1M, 1000, 1000000)
1702 BENCH_REL (boost_shared_ping_pong, burn1M, 1000, 1000000)
1703 BENCH_REL (pthrd_rwlock_ping_pong, burn1M, 1000, 1000000)
1704
1705 // Reproduce with 10 minutes and
1706 //   sudo nice -n -20 \
1707 //     shared_mutex_test --benchmark --bm_min_iters=1000000
1708 //
1709 // Comparison use folly::RWSpinLock as the baseline, with the
1710 // following row being the default SharedMutex (using *Holder or
1711 // Token-ful methods).
1712 // ============================================================================
1713 // folly/experimental/test/SharedMutexTest.cpp     relative  time/iter  iters/s
1714 // ============================================================================
1715 // single_thread_lock_shared_unlock_shared                     22.78ns   43.89M
1716 // single_thread_lock_unlock                                   26.01ns   38.45M
1717 // ----------------------------------------------------------------------------
1718 // ----------------------------------------------------------------------------
1719 // folly_rwspin_reads(1thread)                                 15.09ns   66.25M
1720 // shmtx_wr_pri_reads(1thread)                       69.89%    21.60ns   46.30M
1721 // shmtx_w_bare_reads(1thread)                       58.25%    25.91ns   38.59M
1722 // shmtx_rd_pri_reads(1thread)                       72.50%    20.82ns   48.03M
1723 // shmtx_r_bare_reads(1thread)                       58.27%    25.91ns   38.60M
1724 // folly_ticket_reads(1thread)                       54.80%    27.55ns   36.30M
1725 // boost_shared_reads(1thread)                       10.88%   138.80ns    7.20M
1726 // pthrd_rwlock_reads(1thread)                       40.68%    37.11ns   26.95M
1727 // ----------------------------------------------------------------------------
1728 // folly_rwspin_reads(2thread)                                 92.63ns   10.80M
1729 // shmtx_wr_pri_reads(2thread)                      462.86%    20.01ns   49.97M
1730 // shmtx_w_bare_reads(2thread)                      430.53%    21.51ns   46.48M
1731 // shmtx_rd_pri_reads(2thread)                      487.13%    19.01ns   52.59M
1732 // shmtx_r_bare_reads(2thread)                      433.35%    21.37ns   46.79M
1733 // folly_ticket_reads(2thread)                       69.82%   132.67ns    7.54M
1734 // boost_shared_reads(2thread)                       36.66%   252.63ns    3.96M
1735 // pthrd_rwlock_reads(2thread)                      127.76%    72.50ns   13.79M
1736 // ----------------------------------------------------------------------------
1737 // folly_rwspin_reads(4thread)                                 97.45ns   10.26M
1738 // shmtx_wr_pri_reads(4thread)                      978.22%     9.96ns  100.38M
1739 // shmtx_w_bare_reads(4thread)                      908.35%    10.73ns   93.21M
1740 // shmtx_rd_pri_reads(4thread)                     1032.29%     9.44ns  105.93M
1741 // shmtx_r_bare_reads(4thread)                      912.38%    10.68ns   93.63M
1742 // folly_ticket_reads(4thread)                       46.08%   211.46ns    4.73M
1743 // boost_shared_reads(4thread)                       25.00%   389.74ns    2.57M
1744 // pthrd_rwlock_reads(4thread)                       47.53%   205.01ns    4.88M
1745 // ----------------------------------------------------------------------------
1746 // folly_rwspin_reads(8thread)                                147.24ns    6.79M
1747 // shmtx_wr_pri_reads(8thread)                     2915.66%     5.05ns  198.02M
1748 // shmtx_w_bare_reads(8thread)                     2699.32%     5.45ns  183.32M
1749 // shmtx_rd_pri_reads(8thread)                     3092.58%     4.76ns  210.03M
1750 // shmtx_r_bare_reads(8thread)                     2744.63%     5.36ns  186.40M
1751 // folly_ticket_reads(8thread)                       54.84%   268.47ns    3.72M
1752 // boost_shared_reads(8thread)                       42.40%   347.30ns    2.88M
1753 // pthrd_rwlock_reads(8thread)                       78.90%   186.63ns    5.36M
1754 // ----------------------------------------------------------------------------
1755 // folly_rwspin_reads(16thread)                               166.25ns    6.02M
1756 // shmtx_wr_pri_reads(16thread)                    6133.03%     2.71ns  368.91M
1757 // shmtx_w_bare_reads(16thread)                    5936.05%     2.80ns  357.06M
1758 // shmtx_rd_pri_reads(16thread)                    6786.57%     2.45ns  408.22M
1759 // shmtx_r_bare_reads(16thread)                    5995.54%     2.77ns  360.64M
1760 // folly_ticket_reads(16thread)                      56.35%   295.01ns    3.39M
1761 // boost_shared_reads(16thread)                      51.62%   322.08ns    3.10M
1762 // pthrd_rwlock_reads(16thread)                      92.47%   179.79ns    5.56M
1763 // ----------------------------------------------------------------------------
1764 // folly_rwspin_reads(32thread)                               107.72ns    9.28M
1765 // shmtx_wr_pri_reads(32thread)                    6772.80%     1.59ns  628.77M
1766 // shmtx_w_bare_reads(32thread)                    6236.13%     1.73ns  578.94M
1767 // shmtx_rd_pri_reads(32thread)                    8143.32%     1.32ns  756.00M
1768 // shmtx_r_bare_reads(32thread)                    6485.18%     1.66ns  602.06M
1769 // folly_ticket_reads(32thread)                      35.12%   306.73ns    3.26M
1770 // boost_shared_reads(32thread)                      28.19%   382.17ns    2.62M
1771 // pthrd_rwlock_reads(32thread)                      65.29%   164.99ns    6.06M
1772 // ----------------------------------------------------------------------------
1773 // folly_rwspin_reads(64thread)                               119.46ns    8.37M
1774 // shmtx_wr_pri_reads(64thread)                    6744.92%     1.77ns  564.60M
1775 // shmtx_w_bare_reads(64thread)                    6268.50%     1.91ns  524.72M
1776 // shmtx_rd_pri_reads(64thread)                    7508.56%     1.59ns  628.52M
1777 // shmtx_r_bare_reads(64thread)                    6299.53%     1.90ns  527.32M
1778 // folly_ticket_reads(64thread)                      37.42%   319.26ns    3.13M
1779 // boost_shared_reads(64thread)                      32.58%   366.70ns    2.73M
1780 // pthrd_rwlock_reads(64thread)                      73.64%   162.24ns    6.16M
1781 // ----------------------------------------------------------------------------
1782 // ----------------------------------------------------------------------------
1783 // folly_rwspin(1thread_all_write)                             25.51ns   39.19M
1784 // shmtx_wr_pri(1thread_all_write)                   97.38%    26.20ns   38.17M
1785 // shmtx_rd_pri(1thread_all_write)                   97.55%    26.16ns   38.23M
1786 // folly_ticket(1thread_all_write)                   90.98%    28.04ns   35.66M
1787 // boost_shared(1thread_all_write)                   16.80%   151.89ns    6.58M
1788 // pthrd_rwlock(1thread_all_write)                   63.86%    39.96ns   25.03M
1789 // pthrd_mutex_(1thread_all_write)                   82.05%    31.09ns   32.16M
1790 // ----------------------------------------------------------------------------
1791 // folly_rwspin(2thread_all_write)                            100.70ns    9.93M
1792 // shmtx_wr_pri(2thread_all_write)                   40.83%   246.61ns    4.05M
1793 // shmtx_rd_pri(2thread_all_write)                   40.53%   248.44ns    4.03M
1794 // folly_ticket(2thread_all_write)                   58.49%   172.17ns    5.81M
1795 // boost_shared(2thread_all_write)                   24.26%   415.00ns    2.41M
1796 // pthrd_rwlock(2thread_all_write)                   41.35%   243.49ns    4.11M
1797 // pthrd_mutex_(2thread_all_write)                  146.91%    68.55ns   14.59M
1798 // ----------------------------------------------------------------------------
1799 // folly_rwspin(4thread_all_write)                            199.52ns    5.01M
1800 // shmtx_wr_pri(4thread_all_write)                   51.71%   385.86ns    2.59M
1801 // shmtx_rd_pri(4thread_all_write)                   49.43%   403.62ns    2.48M
1802 // folly_ticket(4thread_all_write)                  117.88%   169.26ns    5.91M
1803 // boost_shared(4thread_all_write)                    9.81%     2.03us  491.48K
1804 // pthrd_rwlock(4thread_all_write)                   28.23%   706.69ns    1.42M
1805 // pthrd_mutex_(4thread_all_write)                  111.54%   178.88ns    5.59M
1806 // ----------------------------------------------------------------------------
1807 // folly_rwspin(8thread_all_write)                            304.61ns    3.28M
1808 // shmtx_wr_pri(8thread_all_write)                   69.77%   436.59ns    2.29M
1809 // shmtx_rd_pri(8thread_all_write)                   66.58%   457.51ns    2.19M
1810 // folly_ticket(8thread_all_write)                  141.00%   216.03ns    4.63M
1811 // boost_shared(8thread_all_write)                    6.11%     4.99us  200.59K
1812 // pthrd_rwlock(8thread_all_write)                   38.03%   800.88ns    1.25M
1813 // pthrd_mutex_(8thread_all_write)                  177.66%   171.45ns    5.83M
1814 // ----------------------------------------------------------------------------
1815 // folly_rwspin(16thread_all_write)                           576.97ns    1.73M
1816 // shmtx_wr_pri(16thread_all_write)                 105.72%   545.77ns    1.83M
1817 // shmtx_rd_pri(16thread_all_write)                 105.13%   548.83ns    1.82M
1818 // folly_ticket(16thread_all_write)                 161.70%   356.82ns    2.80M
1819 // boost_shared(16thread_all_write)                   7.73%     7.46us  134.03K
1820 // pthrd_rwlock(16thread_all_write)                  96.88%   595.54ns    1.68M
1821 // pthrd_mutex_(16thread_all_write)                 330.44%   174.61ns    5.73M
1822 // ----------------------------------------------------------------------------
1823 // folly_rwspin(32thread_all_write)                             1.41us  707.76K
1824 // shmtx_wr_pri(32thread_all_write)                 240.46%   587.58ns    1.70M
1825 // shmtx_rd_pri(32thread_all_write)                 393.71%   358.87ns    2.79M
1826 // folly_ticket(32thread_all_write)                 325.07%   434.65ns    2.30M
1827 // boost_shared(32thread_all_write)                  18.57%     7.61us  131.43K
1828 // pthrd_rwlock(32thread_all_write)                 266.78%   529.62ns    1.89M
1829 // pthrd_mutex_(32thread_all_write)                 877.89%   160.94ns    6.21M
1830 // ----------------------------------------------------------------------------
1831 // folly_rwspin(64thread_all_write)                             1.76us  566.94K
1832 // shmtx_wr_pri(64thread_all_write)                 255.67%   689.91ns    1.45M
1833 // shmtx_rd_pri(64thread_all_write)                 468.82%   376.23ns    2.66M
1834 // folly_ticket(64thread_all_write)                 294.72%   598.49ns    1.67M
1835 // boost_shared(64thread_all_write)                  23.39%     7.54us  132.58K
1836 // pthrd_rwlock(64thread_all_write)                 321.39%   548.83ns    1.82M
1837 // pthrd_mutex_(64thread_all_write)                1165.04%   151.40ns    6.61M
1838 // ----------------------------------------------------------------------------
1839 // ----------------------------------------------------------------------------
1840 // folly_rwspin(1thread_10pct_write)                           19.51ns   51.26M
1841 // shmtx_wr_pri(1thread_10pct_write)                 83.25%    23.43ns   42.67M
1842 // shmtx_rd_pri(1thread_10pct_write)                 83.31%    23.42ns   42.71M
1843 // folly_ticket(1thread_10pct_write)                 70.88%    27.52ns   36.34M
1844 // boost_shared(1thread_10pct_write)                 13.09%   148.99ns    6.71M
1845 // pthrd_rwlock(1thread_10pct_write)                 47.41%    41.15ns   24.30M
1846 // ----------------------------------------------------------------------------
1847 // folly_rwspin(2thread_10pct_write)                          159.42ns    6.27M
1848 // shmtx_wr_pri(2thread_10pct_write)                188.44%    84.60ns   11.82M
1849 // shmtx_rd_pri(2thread_10pct_write)                188.29%    84.67ns   11.81M
1850 // folly_ticket(2thread_10pct_write)                140.28%   113.64ns    8.80M
1851 // boost_shared(2thread_10pct_write)                 42.09%   378.81ns    2.64M
1852 // pthrd_rwlock(2thread_10pct_write)                103.86%   153.49ns    6.51M
1853 // ----------------------------------------------------------------------------
1854 // folly_rwspin(4thread_10pct_write)                          193.35ns    5.17M
1855 // shmtx_wr_pri(4thread_10pct_write)                184.30%   104.91ns    9.53M
1856 // shmtx_rd_pri(4thread_10pct_write)                163.76%   118.07ns    8.47M
1857 // folly_ticket(4thread_10pct_write)                124.07%   155.84ns    6.42M
1858 // boost_shared(4thread_10pct_write)                 16.32%     1.18us  843.92K
1859 // pthrd_rwlock(4thread_10pct_write)                 48.59%   397.94ns    2.51M
1860 // ----------------------------------------------------------------------------
1861 // folly_rwspin(8thread_10pct_write)                          373.17ns    2.68M
1862 // shmtx_wr_pri(8thread_10pct_write)                252.02%   148.08ns    6.75M
1863 // shmtx_rd_pri(8thread_10pct_write)                203.59%   183.30ns    5.46M
1864 // folly_ticket(8thread_10pct_write)                184.37%   202.40ns    4.94M
1865 // boost_shared(8thread_10pct_write)                 15.85%     2.35us  424.72K
1866 // pthrd_rwlock(8thread_10pct_write)                 83.03%   449.45ns    2.22M
1867 // ----------------------------------------------------------------------------
1868 // folly_rwspin(16thread_10pct_write)                         742.87ns    1.35M
1869 // shmtx_wr_pri(16thread_10pct_write)               344.27%   215.78ns    4.63M
1870 // shmtx_rd_pri(16thread_10pct_write)               287.04%   258.80ns    3.86M
1871 // folly_ticket(16thread_10pct_write)               277.25%   267.94ns    3.73M
1872 // boost_shared(16thread_10pct_write)                15.33%     4.85us  206.30K
1873 // pthrd_rwlock(16thread_10pct_write)               158.34%   469.16ns    2.13M
1874 // ----------------------------------------------------------------------------
1875 // folly_rwspin(32thread_10pct_write)                         799.97ns    1.25M
1876 // shmtx_wr_pri(32thread_10pct_write)               351.40%   227.65ns    4.39M
1877 // shmtx_rd_pri(32thread_10pct_write)               341.71%   234.11ns    4.27M
1878 // folly_ticket(32thread_10pct_write)               245.91%   325.31ns    3.07M
1879 // boost_shared(32thread_10pct_write)                 7.72%    10.36us   96.56K
1880 // pthrd_rwlock(32thread_10pct_write)               165.87%   482.30ns    2.07M
1881 // ----------------------------------------------------------------------------
1882 // folly_rwspin(64thread_10pct_write)                           1.12us  892.01K
1883 // shmtx_wr_pri(64thread_10pct_write)               429.84%   260.81ns    3.83M
1884 // shmtx_rd_pri(64thread_10pct_write)               456.93%   245.35ns    4.08M
1885 // folly_ticket(64thread_10pct_write)               219.21%   511.42ns    1.96M
1886 // boost_shared(64thread_10pct_write)                 5.43%    20.65us   48.44K
1887 // pthrd_rwlock(64thread_10pct_write)               233.93%   479.23ns    2.09M
1888 // ----------------------------------------------------------------------------
1889 // ----------------------------------------------------------------------------
1890 // folly_rwspin(1thread_1pct_write)                            18.88ns   52.98M
1891 // shmtx_wr_pri(1thread_1pct_write)                  81.53%    23.15ns   43.19M
1892 // shmtx_w_bare(1thread_1pct_write)                  67.90%    27.80ns   35.97M
1893 // shmtx_rd_pri(1thread_1pct_write)                  81.50%    23.16ns   43.18M
1894 // shmtx_r_bare(1thread_1pct_write)                  67.74%    27.86ns   35.89M
1895 // folly_ticket(1thread_1pct_write)                  68.68%    27.48ns   36.39M
1896 // boost_shared(1thread_1pct_write)                  12.80%   147.51ns    6.78M
1897 // pthrd_rwlock(1thread_1pct_write)                  45.81%    41.20ns   24.27M
1898 // ----------------------------------------------------------------------------
1899 // folly_rwspin(2thread_1pct_write)                           125.85ns    7.95M
1900 // shmtx_wr_pri(2thread_1pct_write)                 359.04%    35.05ns   28.53M
1901 // shmtx_w_bare(2thread_1pct_write)                 475.60%    26.46ns   37.79M
1902 // shmtx_rd_pri(2thread_1pct_write)                 332.75%    37.82ns   26.44M
1903 // shmtx_r_bare(2thread_1pct_write)                 115.64%   108.83ns    9.19M
1904 // folly_ticket(2thread_1pct_write)                 140.24%    89.74ns   11.14M
1905 // boost_shared(2thread_1pct_write)                  40.62%   309.82ns    3.23M
1906 // pthrd_rwlock(2thread_1pct_write)                 134.67%    93.45ns   10.70M
1907 // ----------------------------------------------------------------------------
1908 // folly_rwspin(4thread_1pct_write)                           126.70ns    7.89M
1909 // shmtx_wr_pri(4thread_1pct_write)                 422.20%    30.01ns   33.32M
1910 // shmtx_w_bare(4thread_1pct_write)                 403.52%    31.40ns   31.85M
1911 // shmtx_rd_pri(4thread_1pct_write)                 282.50%    44.85ns   22.30M
1912 // shmtx_r_bare(4thread_1pct_write)                  66.30%   191.10ns    5.23M
1913 // folly_ticket(4thread_1pct_write)                  91.93%   137.83ns    7.26M
1914 // boost_shared(4thread_1pct_write)                  22.74%   557.10ns    1.80M
1915 // pthrd_rwlock(4thread_1pct_write)                  55.66%   227.62ns    4.39M
1916 // ----------------------------------------------------------------------------
1917 // folly_rwspin(8thread_1pct_write)                           169.42ns    5.90M
1918 // shmtx_wr_pri(8thread_1pct_write)                 567.81%    29.84ns   33.51M
1919 // shmtx_w_bare(8thread_1pct_write)                 519.18%    32.63ns   30.64M
1920 // shmtx_rd_pri(8thread_1pct_write)                 172.36%    98.30ns   10.17M
1921 // shmtx_r_bare(8thread_1pct_write)                  75.56%   224.21ns    4.46M
1922 // folly_ticket(8thread_1pct_write)                 104.03%   162.85ns    6.14M
1923 // boost_shared(8thread_1pct_write)                  22.01%   769.73ns    1.30M
1924 // pthrd_rwlock(8thread_1pct_write)                  71.79%   235.99ns    4.24M
1925 // ----------------------------------------------------------------------------
1926 // folly_rwspin(16thread_1pct_write)                          385.88ns    2.59M
1927 // shmtx_wr_pri(16thread_1pct_write)               1039.03%    37.14ns   26.93M
1928 // shmtx_w_bare(16thread_1pct_write)                997.26%    38.69ns   25.84M
1929 // shmtx_rd_pri(16thread_1pct_write)                263.60%   146.39ns    6.83M
1930 // shmtx_r_bare(16thread_1pct_write)                173.16%   222.85ns    4.49M
1931 // folly_ticket(16thread_1pct_write)                179.37%   215.13ns    4.65M
1932 // boost_shared(16thread_1pct_write)                 26.95%     1.43us  698.42K
1933 // pthrd_rwlock(16thread_1pct_write)                166.70%   231.48ns    4.32M
1934 // ----------------------------------------------------------------------------
1935 // folly_rwspin(32thread_1pct_write)                          382.49ns    2.61M
1936 // shmtx_wr_pri(32thread_1pct_write)               1046.64%    36.54ns   27.36M
1937 // shmtx_w_bare(32thread_1pct_write)                922.87%    41.45ns   24.13M
1938 // shmtx_rd_pri(32thread_1pct_write)                251.93%   151.82ns    6.59M
1939 // shmtx_r_bare(32thread_1pct_write)                176.44%   216.78ns    4.61M
1940 // folly_ticket(32thread_1pct_write)                131.07%   291.82ns    3.43M
1941 // boost_shared(32thread_1pct_write)                 12.77%     2.99us  333.95K
1942 // pthrd_rwlock(32thread_1pct_write)                173.43%   220.55ns    4.53M
1943 // ----------------------------------------------------------------------------
1944 // folly_rwspin(64thread_1pct_write)                          510.54ns    1.96M
1945 // shmtx_wr_pri(64thread_1pct_write)               1378.27%    37.04ns   27.00M
1946 // shmtx_w_bare(64thread_1pct_write)               1178.24%    43.33ns   23.08M
1947 // shmtx_rd_pri(64thread_1pct_write)                325.29%   156.95ns    6.37M
1948 // shmtx_r_bare(64thread_1pct_write)                247.82%   206.02ns    4.85M
1949 // folly_ticket(64thread_1pct_write)                117.87%   433.13ns    2.31M
1950 // boost_shared(64thread_1pct_write)                  9.45%     5.40us  185.09K
1951 // pthrd_rwlock(64thread_1pct_write)                236.72%   215.68ns    4.64M
1952 // ----------------------------------------------------------------------------
1953 // folly_rwspin(2thr_2lock_50pct_write)                        10.85ns   92.15M
1954 // shmtx_wr_pri(2thr_2lock_50pct_write)              81.73%    13.28ns   75.32M
1955 // shmtx_rd_pri(2thr_2lock_50pct_write)              81.82%    13.26ns   75.40M
1956 // folly_rwspin(4thr_4lock_50pct_write)                         5.29ns  188.90M
1957 // shmtx_wr_pri(4thr_4lock_50pct_write)              80.89%     6.54ns  152.80M
1958 // shmtx_rd_pri(4thr_4lock_50pct_write)              81.07%     6.53ns  153.14M
1959 // folly_rwspin(8thr_8lock_50pct_write)                         2.63ns  380.57M
1960 // shmtx_wr_pri(8thr_8lock_50pct_write)              80.56%     3.26ns  306.57M
1961 // shmtx_rd_pri(8thr_8lock_50pct_write)              80.29%     3.27ns  305.54M
1962 // folly_rwspin(16thr_16lock_50pct_write)                       1.31ns  764.70M
1963 // shmtx_wr_pri(16thr_16lock_50pct_write)            79.32%     1.65ns  606.54M
1964 // shmtx_rd_pri(16thr_16lock_50pct_write)            79.62%     1.64ns  608.84M
1965 // folly_rwspin(32thr_32lock_50pct_write)                       1.20ns  836.75M
1966 // shmtx_wr_pri(32thr_32lock_50pct_write)            91.67%     1.30ns  767.07M
1967 // shmtx_rd_pri(32thr_32lock_50pct_write)            92.00%     1.30ns  769.82M
1968 // folly_rwspin(64thr_64lock_50pct_write)                       1.39ns  717.80M
1969 // shmtx_wr_pri(64thr_64lock_50pct_write)            93.21%     1.49ns  669.08M
1970 // shmtx_rd_pri(64thr_64lock_50pct_write)            92.49%     1.51ns  663.89M
1971 // ----------------------------------------------------------------------------
1972 // folly_rwspin(2thr_2lock_10pct_write)                        10.24ns   97.70M
1973 // shmtx_wr_pri(2thr_2lock_10pct_write)              76.46%    13.39ns   74.70M
1974 // shmtx_rd_pri(2thr_2lock_10pct_write)              76.35%    13.41ns   74.60M
1975 // folly_rwspin(4thr_4lock_10pct_write)                         5.02ns  199.03M
1976 // shmtx_wr_pri(4thr_4lock_10pct_write)              75.83%     6.63ns  150.91M
1977 // shmtx_rd_pri(4thr_4lock_10pct_write)              76.10%     6.60ns  151.46M
1978 // folly_rwspin(8thr_8lock_10pct_write)                         2.47ns  405.50M
1979 // shmtx_wr_pri(8thr_8lock_10pct_write)              74.54%     3.31ns  302.27M
1980 // shmtx_rd_pri(8thr_8lock_10pct_write)              74.85%     3.29ns  303.52M
1981 // folly_rwspin(16thr_16lock_10pct_write)                       1.22ns  818.68M
1982 // shmtx_wr_pri(16thr_16lock_10pct_write)            73.35%     1.67ns  600.47M
1983 // shmtx_rd_pri(16thr_16lock_10pct_write)            73.38%     1.66ns  600.73M
1984 // folly_rwspin(32thr_32lock_10pct_write)                       1.21ns  827.95M
1985 // shmtx_wr_pri(32thr_32lock_10pct_write)            96.13%     1.26ns  795.89M
1986 // shmtx_rd_pri(32thr_32lock_10pct_write)            96.01%     1.26ns  794.95M
1987 // folly_rwspin(64thr_64lock_10pct_write)                       1.40ns  716.17M
1988 // shmtx_wr_pri(64thr_64lock_10pct_write)            96.91%     1.44ns  694.03M
1989 // shmtx_rd_pri(64thr_64lock_10pct_write)            96.85%     1.44ns  693.64M
1990 // ----------------------------------------------------------------------------
1991 // folly_rwspin(2thr_2lock_1pct_write)                         10.11ns   98.91M
1992 // shmtx_wr_pri(2thr_2lock_1pct_write)               75.07%    13.47ns   74.25M
1993 // shmtx_rd_pri(2thr_2lock_1pct_write)               74.98%    13.48ns   74.16M
1994 // folly_rwspin(4thr_4lock_1pct_write)                          4.96ns  201.77M
1995 // shmtx_wr_pri(4thr_4lock_1pct_write)               74.59%     6.64ns  150.49M
1996 // shmtx_rd_pri(4thr_4lock_1pct_write)               74.60%     6.64ns  150.51M
1997 // folly_rwspin(8thr_8lock_1pct_write)                          2.44ns  410.42M
1998 // shmtx_wr_pri(8thr_8lock_1pct_write)               73.68%     3.31ns  302.41M
1999 // shmtx_rd_pri(8thr_8lock_1pct_write)               73.38%     3.32ns  301.16M
2000 // folly_rwspin(16thr_16lock_1pct_write)                        1.21ns  827.53M
2001 // shmtx_wr_pri(16thr_16lock_1pct_write)             72.11%     1.68ns  596.74M
2002 // shmtx_rd_pri(16thr_16lock_1pct_write)             72.23%     1.67ns  597.73M
2003 // folly_rwspin(32thr_32lock_1pct_write)                        1.22ns  819.53M
2004 // shmtx_wr_pri(32thr_32lock_1pct_write)             98.17%     1.24ns  804.50M
2005 // shmtx_rd_pri(32thr_32lock_1pct_write)             98.21%     1.24ns  804.86M
2006 // folly_rwspin(64thr_64lock_1pct_write)                        1.41ns  710.26M
2007 // shmtx_wr_pri(64thr_64lock_1pct_write)             97.81%     1.44ns  694.71M
2008 // shmtx_rd_pri(64thr_64lock_1pct_write)             99.44%     1.42ns  706.28M
2009 // ----------------------------------------------------------------------------
2010 // ----------------------------------------------------------------------------
2011 // folly_rwspin_ping_pong(burn0)                              641.24ns    1.56M
2012 // shmtx_w_bare_ping_pong(burn0)                     91.07%   704.12ns    1.42M
2013 // shmtx_r_bare_ping_pong(burn0)                     78.70%   814.84ns    1.23M
2014 // folly_ticket_ping_pong(burn0)                     85.67%   748.53ns    1.34M
2015 // boost_shared_ping_pong(burn0)                      5.58%    11.50us   86.96K
2016 // pthrd_rwlock_ping_pong(burn0)                      8.81%     7.28us  137.40K
2017 // ----------------------------------------------------------------------------
2018 // folly_rwspin_ping_pong(burn100k)                           678.97ns    1.47M
2019 // shmtx_w_bare_ping_pong(burn100k)                  99.73%   680.78ns    1.47M
2020 // shmtx_r_bare_ping_pong(burn100k)                  98.67%   688.13ns    1.45M
2021 // folly_ticket_ping_pong(burn100k)                  99.31%   683.68ns    1.46M
2022 // boost_shared_ping_pong(burn100k)                  58.23%     1.17us  857.64K
2023 // pthrd_rwlock_ping_pong(burn100k)                  57.43%     1.18us  845.86K
2024 // ----------------------------------------------------------------------------
2025 // folly_rwspin_ping_pong(burn300k)                             2.03us  492.99K
2026 // shmtx_w_bare_ping_pong(burn300k)                  99.98%     2.03us  492.88K
2027 // shmtx_r_bare_ping_pong(burn300k)                  99.94%     2.03us  492.68K
2028 // folly_ticket_ping_pong(burn300k)                  99.88%     2.03us  492.40K
2029 // boost_shared_ping_pong(burn300k)                  81.43%     2.49us  401.47K
2030 // pthrd_rwlock_ping_pong(burn300k)                  83.22%     2.44us  410.29K
2031 // ----------------------------------------------------------------------------
2032 // folly_rwspin_ping_pong(burn1M)                             677.07ns    1.48M
2033 // shmtx_w_bare_ping_pong(burn1M)                   100.50%   673.74ns    1.48M
2034 // shmtx_r_bare_ping_pong(burn1M)                   100.14%   676.12ns    1.48M
2035 // folly_ticket_ping_pong(burn1M)                   100.44%   674.14ns    1.48M
2036 // boost_shared_ping_pong(burn1M)                    93.04%   727.72ns    1.37M
2037 // pthrd_rwlock_ping_pong(burn1M)                    94.52%   716.30ns    1.40M
2038 // ============================================================================
2039
2040 int main(int argc, char** argv) {
2041   (void)folly_rwspin_reads;
2042   (void)shmtx_wr_pri_reads;
2043   (void)shmtx_w_bare_reads;
2044   (void)shmtx_rd_pri_reads;
2045   (void)shmtx_r_bare_reads;
2046   (void)folly_ticket_reads;
2047   (void)boost_shared_reads;
2048   (void)pthrd_rwlock_reads;
2049   (void)folly_rwspin;
2050   (void)shmtx_wr_pri;
2051   (void)shmtx_w_bare;
2052   (void)shmtx_rd_pri;
2053   (void)shmtx_r_bare;
2054   (void)folly_ticket;
2055   (void)boost_shared;
2056   (void)pthrd_rwlock;
2057   (void)pthrd_mutex_;
2058   (void)folly_rwspin_ping_pong;
2059   (void)shmtx_w_bare_ping_pong;
2060   (void)shmtx_r_bare_ping_pong;
2061   (void)folly_ticket_ping_pong;
2062   (void)boost_shared_ping_pong;
2063   (void)pthrd_rwlock_ping_pong;
2064
2065   testing::InitGoogleTest(&argc, argv);
2066   gflags::ParseCommandLineFlags(&argc, &argv, true);
2067   int rv = RUN_ALL_TESTS();
2068   folly::runBenchmarksOnFlag();
2069   return rv;
2070 }