cds/compiler/cxx11_atomic.h

   1 //$$CDS-header$$
   2
   3 #ifndef CDSLIB_COMPILER_CXX11_ATOMIC_H
   4 #define CDSLIB_COMPILER_CXX11_ATOMIC_H
   5 //@cond
   6
   7 #include <type_traits>  // make_unsigned
   8 #include <cds/details/defs.h>
   9 #include <cds/details/aligned_type.h>
  10
  11 namespace cds { namespace cxx11_atomic {
  12     typedef enum memory_order {
  13         memory_order_relaxed,
  14         memory_order_consume,
  15         memory_order_acquire,
  16         memory_order_release,
  17         memory_order_acq_rel,
  18         memory_order_seq_cst
  19     } memory_order;
  20
  21 }}  // namespace cds::cxx11_atomic
  22
  23
  24 #if CDS_COMPILER == CDS_COMPILER_MSVC || (CDS_COMPILER == CDS_COMPILER_INTEL && CDS_OS_INTERFACE == CDS_OSI_WINDOWS)
  25 #   if CDS_PROCESSOR_ARCH == CDS_PROCESSOR_X86
  26 #       include <cds/compiler/vc/x86/cxx11_atomic.h>
  27 #   elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_AMD64
  28 #       include <cds/compiler/vc/amd64/cxx11_atomic.h>
  29 #   else
  30 #       error "MS VC++ compiler: unsupported processor architecture"
  31 #   endif
  32 #elif CDS_COMPILER == CDS_COMPILER_GCC || CDS_COMPILER == CDS_COMPILER_CLANG || CDS_COMPILER == CDS_COMPILER_INTEL
  33 #   if CDS_PROCESSOR_ARCH == CDS_PROCESSOR_X86
  34 #       include <cds/compiler/gcc/x86/cxx11_atomic.h>
  35 #   elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_AMD64
  36 #       include <cds/compiler/gcc/amd64/cxx11_atomic.h>
  37 #   elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_IA64
  38 #       include <cds/compiler/gcc/ia64/cxx11_atomic.h>
  39 #   elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_SPARC
  40 #       include <cds/compiler/gcc/sparc/cxx11_atomic.h>
  41 #   elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_PPC64
  42 #       include <cds/compiler/gcc/ppc64/cxx11_atomic.h>
  43 //#   elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_ARM7
  44 //#       include <cds/compiler/gcc/arm7/cxx11_atomic.h>
  45 #   else
  46 #       error "GCC compiler: unsupported processor architecture. Try to use native C++11 atomic or boost.atomic"
  47 #   endif
  48 #else
  49 #   error "Undefined compiler"
  50 #endif
  51
  52 namespace cds { namespace cxx11_atomic {
  53
  54     // forward declarations
  55     template <class T>
  56     struct atomic;
  57
  58     namespace details {
  59
  60         template <typename T, size_t Size, typename Primary = T >
  61         struct atomic_generic_ops;
  62
  63         template <typename T, size_t Size>
  64         struct atomic_integral_ops;
  65
  66         template <size_t TypeSize>
  67         struct primary_type;
  68
  69         template <>
  70         struct primary_type<1>
  71         {
  72             typedef std::uint8_t type;
  73         };
  74         template <>
  75         struct primary_type<2>
  76         {
  77             typedef std::uint16_t type;
  78         };
  79         template <>
  80         struct primary_type<4>
  81         {
  82             typedef std::uint32_t type;
  83         };
  84         template <>
  85         struct primary_type<8>
  86         {
  87             typedef std::uint64_t type;
  88         };
  89
  90         template <typename T, typename Primary>
  91         struct make_atomic_primary
  92         {
  93             typedef T       source_type;
  94             typedef Primary primary_type;
  95
  96             static primary_type volatile * ptr( source_type volatile * p ) CDS_NOEXCEPT
  97             {
  98                 return reinterpret_cast<primary_type volatile *>(p);
  99             }
 100             static primary_type const volatile * ptr( source_type const volatile * p ) CDS_NOEXCEPT
 101             {
 102                 return reinterpret_cast<primary_type const volatile *>(p);
 103             }
 104
 105             static primary_type val( source_type v ) CDS_NOEXCEPT
 106             {
 107                 return *reinterpret_cast<primary_type*>(&v);
 108             }
 109
 110             static primary_type& ref( source_type& v ) CDS_NOEXCEPT
 111             {
 112                 return reinterpret_cast<primary_type&>(v);
 113             }
 114
 115             static primary_type const& ref( source_type const& v ) CDS_NOEXCEPT
 116             {
 117                 return reinterpret_cast<primary_type const&>(v);
 118             }
 119
 120             static source_type ret( primary_type r ) CDS_NOEXCEPT
 121             {
 122                 return *reinterpret_cast<source_type *>(&r);
 123             }
 124         };
 125
 126         template <typename T>
 127         struct make_atomic_primary<T, T>
 128         {
 129             typedef T source_type;
 130             typedef T primary_type;
 131
 132             static primary_type volatile * ptr( source_type volatile * p ) CDS_NOEXCEPT
 133             {
 134                 return p;
 135             }
 136             static primary_type const volatile * ptr( source_type const volatile * p ) CDS_NOEXCEPT
 137             {
 138                 return p;
 139             }
 140
 141             static primary_type val( source_type v ) CDS_NOEXCEPT
 142             {
 143                 return v;
 144             }
 145
 146             static primary_type& ref( source_type& v ) CDS_NOEXCEPT
 147             {
 148                 return v;
 149             }
 150
 151             static source_type ret( primary_type r ) CDS_NOEXCEPT
 152             {
 153                 return r;
 154             }
 155         };
 156
 157         template <typename T>
 158         struct atomic_integral_bitwise_ops
 159         {
 160         public:
 161             typedef typename std::make_unsigned<T>::type unsigned_type;
 162             typedef atomic_generic_ops<unsigned_type, sizeof(unsigned_type)> atomic_ops;
 163
 164             static T fetch_and(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 165             {
 166                 unsigned_type cur = atomic_ops::atomic_load_explicit( reinterpret_cast<unsigned_type volatile *>(pDest), memory_order_relaxed );
 167                 do {} while ( !atomic_ops::atomic_compare_exchange_weak_explicit(
 168                     reinterpret_cast<unsigned_type volatile *>(pDest), &cur, cur & unsigned_type(val), order, memory_order_relaxed ));
 169                 return T(cur);
 170             }
 171
 172             static T fetch_or(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 173             {
 174                 unsigned_type cur = atomic_ops::atomic_load_explicit( reinterpret_cast<unsigned_type volatile *>(pDest), memory_order_relaxed );
 175                 do {} while ( !atomic_ops::atomic_compare_exchange_weak_explicit(
 176                     reinterpret_cast<unsigned_type volatile *>(pDest), &cur, cur | unsigned_type(val), order, memory_order_relaxed ));
 177                 return T(cur);
 178             }
 179
 180             static T fetch_xor(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 181             {
 182                 unsigned_type cur = atomic_ops::atomic_load_explicit( reinterpret_cast<unsigned_type volatile *>(pDest), memory_order_relaxed );
 183                 do {} while ( !atomic_ops::atomic_compare_exchange_weak_explicit(
 184                     reinterpret_cast<unsigned_type volatile *>(pDest), &cur, cur ^ unsigned_type(val), order, memory_order_relaxed ));
 185                 return T(cur);
 186             }
 187         };
 188
 189
 190         // 8-bit atomic operations
 191
 192         template <typename T, typename Primary>
 193         struct atomic_generic_ops< T, 1, Primary >
 194         {
 195             typedef make_atomic_primary<T, Primary> primary;
 196
 197             // store
 198             static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
 199             {
 200                 platform::store8( primary::ptr(pDest), primary::val(v), order );
 201             }
 202             static void atomic_store_explicit( T * pDest, T v, memory_order order ) CDS_NOEXCEPT
 203             {
 204                 platform::store8( primary::ptr(pDest), primary::val(v), order );
 205             }
 206             static void atomic_store( T volatile * pDest, T v ) CDS_NOEXCEPT
 207             {
 208                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 209             }
 210             static void atomic_store( T * pDest, T v ) CDS_NOEXCEPT
 211             {
 212                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 213             }
 214
 215             // load
 216             static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
 217             {
 218                 return primary::ret( platform::load8( primary::ptr(pSrc), order ));
 219             }
 220             static T atomic_load_explicit( T const * pSrc, memory_order order ) CDS_NOEXCEPT
 221             {
 222                 return primary::ret( platform::load8( primary::ptr(pSrc), order ));
 223             }
 224             static T atomic_load( T volatile const * pSrc ) CDS_NOEXCEPT
 225             {
 226                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 227             }
 228             static T atomic_load( T const * pSrc ) CDS_NOEXCEPT
 229             {
 230                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 231             }
 232
 233             // exchange
 234             static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
 235             {
 236                 return primary::ret( platform::exchange8( primary::ptr(pDest), primary::val(val), order ));
 237             }
 238             static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) CDS_NOEXCEPT
 239             {
 240                 return primary::ret( platform::exchange8( primary::ptr(pDest), primary::val(val), order ));
 241             }
 242             static T atomic_exchange( T volatile * pDest, T val ) CDS_NOEXCEPT
 243             {
 244                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 245             }
 246             static T atomic_exchange( T * pDest, T val ) CDS_NOEXCEPT
 247             {
 248                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 249             }
 250
 251             // cas
 252             static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 253             {
 254                 assert( expected );
 255                 return platform::cas8_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 256             }
 257             static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 258             {
 259                 assert( expected );
 260                 return platform::cas8_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 261             }
 262             static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 263             {
 264                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 265             }
 266             static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 267             {
 268                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 269             }
 270             static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 271             {
 272                 assert( expected );
 273                 return platform::cas8_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 274             }
 275             static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 276             {
 277                 assert( expected );
 278                 return platform::cas8_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 279             }
 280             static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 281             {
 282                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 283             }
 284             static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 285             {
 286                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 287             }
 288         };
 289
 290         template <typename T>
 291         struct atomic_integral_ops< T, 1 >
 292             : atomic_generic_ops<T, 1, T >
 293             , atomic_integral_bitwise_ops<T>
 294         {
 295             typedef atomic_integral_bitwise_ops<T> bitwise_ops;
 296
 297             // fetch_add
 298             static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 299             {
 300 #           ifdef CDS_ATOMIC_fetch8_add_defined
 301                 return platform::fetch8_add( pDest, val, order );
 302 #           else
 303                 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
 304                 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
 305                 return cur;
 306 #           endif
 307             }
 308             static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 309             {
 310                 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 311             }
 312             static T atomic_fetch_add( T volatile * pDest, T val ) CDS_NOEXCEPT
 313             {
 314                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 315             }
 316             static T atomic_fetch_add( T * pDest, T val ) CDS_NOEXCEPT
 317             {
 318                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 319             }
 320
 321             // fetch_sub
 322             static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 323             {
 324 #           ifdef CDS_ATOMIC_fetch8_sub_defined
 325                 return platform::fetch8_sub( pDest, val, order );
 326 #           else
 327                 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
 328                 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
 329                 return cur;
 330 #           endif
 331             }
 332             static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 333             {
 334                 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 335             }
 336             static T atomic_fetch_sub( T volatile * pDest, T val ) CDS_NOEXCEPT
 337             {
 338                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
 339             }
 340             static T atomic_fetch_sub( T * pDest, T val ) CDS_NOEXCEPT
 341             {
 342                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
 343             }
 344
 345             // fetch_and
 346             static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 347             {
 348 #           ifdef CDS_ATOMIC_fetch8_and_defined
 349                 return platform::fetch8_and( pDest, val, order );
 350 #           else
 351                 return bitwise_ops::fetch_and( pDest, val, order );
 352 #           endif
 353             }
 354             static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 355             {
 356                 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 357             }
 358             static T atomic_fetch_and( T volatile * pDest, T val ) CDS_NOEXCEPT
 359             {
 360                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
 361             }
 362             static T atomic_fetch_and( T * pDest, T val ) CDS_NOEXCEPT
 363             {
 364                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
 365             }
 366
 367             // fetch_or
 368             static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 369             {
 370 #           ifdef CDS_ATOMIC_fetch8_or_defined
 371                 return platform::fetch8_or( pDest, val, order );
 372 #           else
 373                 return bitwise_ops::fetch_or( pDest, val, order );
 374 #           endif
 375             }
 376             static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 377             {
 378                 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 379             }
 380             static T atomic_fetch_or( T volatile * pDest, T val ) CDS_NOEXCEPT
 381             {
 382                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
 383             }
 384             static T atomic_fetch_or( T * pDest, T val ) CDS_NOEXCEPT
 385             {
 386                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
 387             }
 388
 389             // fetch_xor
 390             static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 391             {
 392 #           ifdef CDS_ATOMIC_fetch8_xor_defined
 393                 return platform::fetch8_xor( pDest, val, order );
 394 #           else
 395                 return bitwise_ops::fetch_xor( pDest, val, order );
 396 #           endif
 397             }
 398             static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 399             {
 400                 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 401             }
 402             static T atomic_fetch_xor( T volatile * pDest, T val ) CDS_NOEXCEPT
 403             {
 404                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
 405             }
 406             static T atomic_fetch_xor( T * pDest, T val ) CDS_NOEXCEPT
 407             {
 408                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
 409             }
 410         };
 411
 412         // 16-bit atomic operations
 413
 414         template <typename T, typename Primary>
 415         struct atomic_generic_ops< T, 2, Primary >
 416         {
 417             typedef make_atomic_primary<T, Primary> primary;
 418
 419             // store
 420             static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
 421             {
 422                 platform::store16( primary::ptr(pDest), primary::val(v), order );
 423             }
 424             static void atomic_store_explicit( T * pDest, T v, memory_order order ) CDS_NOEXCEPT
 425             {
 426                 platform::store16( primary::ptr(pDest), primary::val(v), order );
 427             }
 428             static void atomic_store( T volatile * pDest, T v ) CDS_NOEXCEPT
 429             {
 430                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 431             }
 432             static void atomic_store( T * pDest, T v ) CDS_NOEXCEPT
 433             {
 434                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 435             }
 436
 437             // load
 438             static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
 439             {
 440                 return primary::ret( platform::load16( primary::ptr(pSrc), order ));
 441             }
 442             static T atomic_load_explicit( T const * pSrc, memory_order order ) CDS_NOEXCEPT
 443             {
 444                 return primary::ret( platform::load16( primary::ptr(pSrc), order ));
 445             }
 446             static T atomic_load( T volatile const * pSrc ) CDS_NOEXCEPT
 447             {
 448                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 449             }
 450             static T atomic_load( T const * pSrc ) CDS_NOEXCEPT
 451             {
 452                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 453             }
 454
 455             // exchange
 456             static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
 457             {
 458                 return primary::ret( platform::exchange16( primary::ptr(pDest), primary::val(val), order ));
 459             }
 460             static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) CDS_NOEXCEPT
 461             {
 462                 return primary::ret( platform::exchange16( primary::ptr(pDest), primary::val(val), order ));
 463             }
 464             static T atomic_exchange( T volatile * pDest, T val ) CDS_NOEXCEPT
 465             {
 466                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 467             }
 468             static T atomic_exchange( T * pDest, T val ) CDS_NOEXCEPT
 469             {
 470                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 471             }
 472
 473             // cas
 474             static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 475             {
 476                 assert( expected );
 477                 return platform::cas16_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 478             }
 479             static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 480             {
 481                 assert( expected );
 482                 return platform::cas16_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 483             }
 484             static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 485             {
 486                 return atomic_compare_exchange_weak_explicit( pDest, expected, primary::val(desired), memory_order_seq_cst, memory_order_relaxed );
 487             }
 488             static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 489             {
 490                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 491             }
 492             static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 493             {
 494                 assert( expected );
 495                 return platform::cas16_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 496             }
 497             static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 498             {
 499                 assert( expected );
 500                 return platform::cas16_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 501             }
 502             static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 503             {
 504                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 505             }
 506             static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 507             {
 508                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 509             }
 510         };
 511
 512         template <typename T>
 513         struct atomic_integral_ops< T, 2 >
 514             : atomic_generic_ops< T, 2, T >
 515             , atomic_integral_bitwise_ops<T>
 516         {
 517             typedef atomic_integral_bitwise_ops<T> bitwise_ops;
 518
 519             // fetch_add
 520             static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 521             {
 522 #           ifdef CDS_ATOMIC_fetch16_add_defined
 523                 return platform::fetch16_add( pDest, val, order );
 524 #           else
 525                 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
 526                 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
 527                 return cur;
 528 #           endif
 529             }
 530             static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 531             {
 532                 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 533             }
 534             static T atomic_fetch_add( T volatile * pDest, T val ) CDS_NOEXCEPT
 535             {
 536                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 537             }
 538             static T atomic_fetch_add( T * pDest, T val ) CDS_NOEXCEPT
 539             {
 540                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 541             }
 542
 543             // fetch_sub
 544             static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 545             {
 546 #           ifdef CDS_ATOMIC_fetch16_sub_defined
 547                 return platform::fetch16_sub( pDest, val, order );
 548 #           else
 549                 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
 550                 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
 551                 return cur;
 552 #           endif
 553             }
 554             static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 555             {
 556                 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 557             }
 558             static T atomic_fetch_sub( T volatile * pDest, T val ) CDS_NOEXCEPT
 559             {
 560                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
 561             }
 562             static T atomic_fetch_sub( T * pDest, T val ) CDS_NOEXCEPT
 563             {
 564                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
 565             }
 566
 567             // fetch_and
 568             static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 569             {
 570 #           ifdef CDS_ATOMIC_fetch16_and_defined
 571                 return platform::fetch16_and( pDest, val, order );
 572 #           else
 573                 return bitwise_ops::fetch_and( pDest, val, order );
 574 #           endif
 575             }
 576             static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 577             {
 578                 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 579             }
 580             static T atomic_fetch_and( T volatile * pDest, T val ) CDS_NOEXCEPT
 581             {
 582                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
 583             }
 584             static T atomic_fetch_and( T * pDest, T val ) CDS_NOEXCEPT
 585             {
 586                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
 587             }
 588
 589             // fetch_or
 590             static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 591             {
 592 #           ifdef CDS_ATOMIC_fetch16_or_defined
 593                 return platform::fetch16_or( pDest, val, order );
 594 #           else
 595                 return bitwise_ops::fetch_or( pDest, val, order );
 596 #           endif
 597             }
 598             static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 599             {
 600                 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 601             }
 602             static T atomic_fetch_or( T volatile * pDest, T val ) CDS_NOEXCEPT
 603             {
 604                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
 605             }
 606             static T atomic_fetch_or( T * pDest, T val ) CDS_NOEXCEPT
 607             {
 608                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
 609             }
 610
 611             // fetch_xor
 612             static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 613             {
 614 #           ifdef CDS_ATOMIC_fetch16_xor_defined
 615                 return platform::fetch16_xor( pDest, val, order );
 616 #           else
 617                 return bitwise_ops::fetch_xor( pDest, val, order );
 618 #           endif
 619             }
 620             static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 621             {
 622                 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 623             }
 624             static T atomic_fetch_xor( T volatile * pDest, T val ) CDS_NOEXCEPT
 625             {
 626                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
 627             }
 628             static T atomic_fetch_xor( T * pDest, T val ) CDS_NOEXCEPT
 629             {
 630                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
 631             }
 632         };
 633
 634         // 32-bit atomic operations
 635
 636         template <typename T, typename Primary>
 637         struct atomic_generic_ops< T, 4, Primary >
 638         {
 639             typedef make_atomic_primary<T, Primary> primary;
 640
 641             // store
 642             static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
 643             {
 644                 platform::store32( primary::ptr(pDest), primary::val(v), order );
 645             }
 646             static void atomic_store_explicit( T * pDest, T v, memory_order order ) CDS_NOEXCEPT
 647             {
 648                 platform::store32( primary::ptr(pDest), primary::val(v), order );
 649             }
 650             static void atomic_store( T volatile * pDest, T v ) CDS_NOEXCEPT
 651             {
 652                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 653             }
 654             static void atomic_store( T * pDest, T v ) CDS_NOEXCEPT
 655             {
 656                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 657             }
 658
 659             // load
 660             static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
 661             {
 662                 return primary::ret( platform::load32( primary::ptr(pSrc), order ));
 663             }
 664             static T atomic_load_explicit( T const * pSrc, memory_order order ) CDS_NOEXCEPT
 665             {
 666                 return primary::ret( platform::load32( primary::ptr(pSrc), order ));
 667             }
 668             static T atomic_load( T volatile const * pSrc ) CDS_NOEXCEPT
 669             {
 670                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 671             }
 672             static T atomic_load( T const * pSrc ) CDS_NOEXCEPT
 673             {
 674                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 675             }
 676
 677             // exchange
 678             static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
 679             {
 680                 return primary::ret( platform::exchange32( primary::ptr(pDest), primary::val(val), order ));
 681             }
 682             static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) CDS_NOEXCEPT
 683             {
 684                 return primary::ret( platform::exchange32( primary::ptr(pDest), primary::val(val), order ));
 685             }
 686             static T atomic_exchange( T volatile * pDest, T val ) CDS_NOEXCEPT
 687             {
 688                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 689             }
 690             static T atomic_exchange( T * pDest, T val ) CDS_NOEXCEPT
 691             {
 692                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 693             }
 694
 695             // cas
 696             static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 697             {
 698                 assert( expected );
 699                 return platform::cas32_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 700             }
 701             static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 702             {
 703                 assert( expected );
 704                 return platform::cas32_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 705             }
 706             static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 707             {
 708                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 709             }
 710             static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 711             {
 712                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 713             }
 714             static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 715             {
 716                 assert( expected );
 717                 return platform::cas32_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 718             }
 719             static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 720             {
 721                 assert( expected );
 722                 return platform::cas32_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 723             }
 724             static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 725             {
 726                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 727             }
 728             static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 729             {
 730                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 731             }
 732         };
 733
 734         template <typename T>
 735         struct atomic_integral_ops< T, 4 >
 736             : atomic_generic_ops< T, 4, T >
 737             , atomic_integral_bitwise_ops<T>
 738         {
 739             typedef atomic_integral_bitwise_ops<T> bitwise_ops;
 740             // fetch_add
 741             static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 742             {
 743 #           ifdef CDS_ATOMIC_fetch32_add_defined
 744                 return platform::fetch32_add( pDest, val, order );
 745 #           else
 746                 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
 747                 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
 748                 return cur;
 749 #           endif
 750             }
 751             static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 752             {
 753                 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 754             }
 755             static T atomic_fetch_add( T volatile * pDest, T val ) CDS_NOEXCEPT
 756             {
 757                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 758             }
 759             static T atomic_fetch_add( T * pDest, T val ) CDS_NOEXCEPT
 760             {
 761                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 762             }
 763
 764             // fetch_sub
 765             static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 766             {
 767 #           ifdef CDS_ATOMIC_fetch32_sub_defined
 768                 return platform::fetch32_sub( pDest, val, order );
 769 #           else
 770                 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
 771                 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
 772                 return cur;
 773 #           endif
 774             }
 775             static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 776             {
 777                 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 778             }
 779             static T atomic_fetch_sub( T volatile * pDest, T val ) CDS_NOEXCEPT
 780             {
 781                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
 782             }
 783             static T atomic_fetch_sub( T * pDest, T val ) CDS_NOEXCEPT
 784             {
 785                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
 786             }
 787
 788             // fetch_and
 789             static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 790             {
 791 #           ifdef CDS_ATOMIC_fetch32_and_defined
 792                 return platform::fetch32_and( pDest, val, order );
 793 #           else
 794                 return bitwise_ops::fetch_and( pDest, val, order );
 795 #           endif
 796             }
 797             static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 798             {
 799                 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 800             }
 801             static T atomic_fetch_and( T volatile * pDest, T val ) CDS_NOEXCEPT
 802             {
 803                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
 804             }
 805             static T atomic_fetch_and( T * pDest, T val ) CDS_NOEXCEPT
 806             {
 807                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
 808             }
 809
 810             // fetch_or
 811             static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 812             {
 813 #           ifdef CDS_ATOMIC_fetch32_or_defined
 814                 return platform::fetch32_or( pDest, val, order );
 815 #           else
 816                 return bitwise_ops::fetch_or( pDest, val, order );
 817 #           endif
 818             }
 819             static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 820             {
 821                 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 822             }
 823             static T atomic_fetch_or( T volatile * pDest, T val ) CDS_NOEXCEPT
 824             {
 825                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
 826             }
 827             static T atomic_fetch_or( T * pDest, T val ) CDS_NOEXCEPT
 828             {
 829                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
 830             }
 831
 832             // fetch_xor
 833             static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 834             {
 835 #           ifdef CDS_ATOMIC_fetch32_xor_defined
 836                 return platform::fetch32_xor( pDest, val, order );
 837 #           else
 838                 return bitwise_ops::fetch_xor( pDest, val, order );
 839 #           endif
 840             }
 841             static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 842             {
 843                 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 844             }
 845             static T atomic_fetch_xor( T volatile * pDest, T val ) CDS_NOEXCEPT
 846             {
 847                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
 848             }
 849             static T atomic_fetch_xor( T * pDest, T val ) CDS_NOEXCEPT
 850             {
 851                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
 852             }
 853         };
 854
 855
 856         // 64-bit atomic operations
 857
 858         template <typename T, typename Primary>
 859         struct atomic_generic_ops< T, 8, Primary >
 860         {
 861             typedef make_atomic_primary<T, Primary> primary;
 862
 863             // store
 864             static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
 865             {
 866                 platform::store64( primary::ptr(pDest), primary::val(v), order );
 867             }
 868             static void atomic_store_explicit( T * pDest, T v, memory_order order ) CDS_NOEXCEPT
 869             {
 870                 platform::store64( primary::ptr(pDest), primary::val(v), order );
 871             }
 872             static void atomic_store( T volatile * pDest, T v ) CDS_NOEXCEPT
 873             {
 874                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 875             }
 876             static void atomic_store( T * pDest, T v ) CDS_NOEXCEPT
 877             {
 878                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 879             }
 880
 881             // load
 882             static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
 883             {
 884                 return primary::ret( platform::load64( primary::ptr(pSrc), order ));
 885             }
 886             static T atomic_load_explicit( T const * pSrc, memory_order order ) CDS_NOEXCEPT
 887             {
 888                 return primary::ret( platform::load64( primary::ptr(pSrc), order ));
 889             }
 890             static T atomic_load( T volatile const * pSrc ) CDS_NOEXCEPT
 891             {
 892                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 893             }
 894             static T atomic_load( T const * pSrc ) CDS_NOEXCEPT
 895             {
 896                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 897             }
 898
 899             // exchange
 900             static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
 901             {
 902                 return primary::ret( platform::exchange64( primary::ptr(pDest), primary::val(val), order ));
 903             }
 904             static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) CDS_NOEXCEPT
 905             {
 906                 return primary::ret( platform::exchange64( primary::ptr(pDest), primary::val(val), order ));
 907             }
 908             static T atomic_exchange( T volatile * pDest, T val ) CDS_NOEXCEPT
 909             {
 910                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 911             }
 912             static T atomic_exchange( T * pDest, T val ) CDS_NOEXCEPT
 913             {
 914                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 915             }
 916
 917             // cas
 918             static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 919             {
 920                 assert( expected );
 921                 return platform::cas64_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 922             }
 923             static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 924             {
 925                 assert( expected );
 926                 return platform::cas64_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 927             }
 928             static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 929             {
 930                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 931             }
 932             static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 933             {
 934                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 935             }
 936             static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 937             {
 938                 assert( expected );
 939                 return platform::cas64_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 940             }
 941             static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 942             {
 943                 assert( expected );
 944                 return platform::cas64_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 945             }
 946             static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 947             {
 948                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 949             }
 950             static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 951             {
 952                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 953             }
 954         };
 955
 956
 957         template <typename T>
 958         struct atomic_integral_ops< T, 8 >
 959             : atomic_generic_ops< T, 8, T >
 960             , atomic_integral_bitwise_ops<T>
 961         {
 962             typedef atomic_integral_bitwise_ops<T>  bitwise_ops;
 963             typedef atomic_generic_ops<T, 8, T>     general_ops;
 964
 965             // fetch_add
 966             static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 967             {
 968 #           ifdef CDS_ATOMIC_fetch64_add_defined
 969                 return platform::fetch64_add( pDest, val, order );
 970 #           else
 971                 T cur = general_ops::atomic_load_explicit( pDest, memory_order_relaxed );
 972                 do {} while ( !general_ops::atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
 973                 return cur;
 974 #           endif
 975             }
 976             static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 977             {
 978                 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 979             }
 980             static T atomic_fetch_add( T volatile * pDest, T val ) CDS_NOEXCEPT
 981             {
 982                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 983             }
 984             static T atomic_fetch_add( T * pDest, T val ) CDS_NOEXCEPT
 985             {
 986                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 987             }
 988
 989             // fetch_sub
 990             static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 991             {
 992 #           ifdef CDS_ATOMIC_fetch64_sub_defined
 993                 return platform::fetch64_sub( pDest, val, order );
 994 #           else
 995                 T cur = general_ops::atomic_load_explicit( pDest, memory_order_relaxed );
 996                 do {} while ( !general_ops::atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
 997                 return cur;
 998 #           endif
 999             }
1000             static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
1001             {
1002                 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1003             }
1004             static T atomic_fetch_sub( T volatile * pDest, T val ) CDS_NOEXCEPT
1005             {
1006                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1007             }
1008             static T atomic_fetch_sub( T * pDest, T val ) CDS_NOEXCEPT
1009             {
1010                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1011             }
1012
1013             // fetch_and
1014             static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
1015             {
1016 #           ifdef CDS_ATOMIC_fetch64_and_defined
1017                 return platform::fetch64_and( pDest, val, order );
1018 #           else
1019                 return bitwise_ops::fetch_and( pDest, val, order );
1020 #           endif
1021             }
1022             static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
1023             {
1024                 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1025             }
1026             static T atomic_fetch_and( T volatile * pDest, T val ) CDS_NOEXCEPT
1027             {
1028                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
1029             }
1030             static T atomic_fetch_and( T * pDest, T val ) CDS_NOEXCEPT
1031             {
1032                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
1033             }
1034
1035             // fetch_or
1036             static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
1037             {
1038 #           ifdef CDS_ATOMIC_fetch64_or_defined
1039                 return platform::fetch64_or( pDest, val, order );
1040 #           else
1041                 return bitwise_ops::fetch_or( pDest, val, order );
1042 #           endif
1043             }
1044             static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
1045             {
1046                 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1047             }
1048             static T atomic_fetch_or( T volatile * pDest, T val ) CDS_NOEXCEPT
1049             {
1050                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
1051             }
1052             static T atomic_fetch_or( T * pDest, T val ) CDS_NOEXCEPT
1053             {
1054                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
1055             }
1056
1057             // fetch_xor
1058             static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
1059             {
1060 #           ifdef CDS_ATOMIC_fetch64_xor_defined
1061                 return platform::fetch64_xor( pDest, val, order );
1062 #           else
1063                 return bitwise_ops::fetch_xor( pDest, val, order );
1064 #           endif
1065             }
1066             static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
1067             {
1068                 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1069             }
1070             static T atomic_fetch_xor( T volatile * pDest, T val ) CDS_NOEXCEPT
1071             {
1072                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
1073             }
1074             static T atomic_fetch_xor( T * pDest, T val ) CDS_NOEXCEPT
1075             {
1076                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
1077             }
1078         };
1079
1080
1081         // atomic pointer operations
1082         template <typename T>
1083         struct atomic_pointer_base
1084         {
1085             // store
1086             static void atomic_store_explicit( T * volatile * pDest, T * v, memory_order order ) CDS_NOEXCEPT
1087             {
1088                 platform::store_ptr( pDest, v, order );
1089             }
1090             static void atomic_store_explicit( T * * pDest, T * v, memory_order order ) CDS_NOEXCEPT
1091             {
1092                 platform::store_ptr( pDest, v, order );
1093             }
1094             static void atomic_store( T * volatile * pDest, T * v ) CDS_NOEXCEPT
1095             {
1096                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
1097             }
1098             static void atomic_store( T * * pDest, T * v ) CDS_NOEXCEPT
1099             {
1100                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
1101             }
1102
1103             // load
1104             static T * atomic_load_explicit( T * volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
1105             {
1106                 return platform::load_ptr( pSrc, order );
1107             }
1108             static T * atomic_load_explicit( T * const * pSrc, memory_order order ) CDS_NOEXCEPT
1109             {
1110                 return platform::load_ptr( pSrc, order );
1111             }
1112             static T * atomic_load( T * volatile const * pSrc ) CDS_NOEXCEPT
1113             {
1114                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
1115             }
1116             static T * atomic_load( T * const * pSrc ) CDS_NOEXCEPT
1117             {
1118                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
1119             }
1120
1121             // exchange
1122             static T * atomic_exchange_explicit( T * volatile * pDest, T * val, memory_order order ) CDS_NOEXCEPT
1123             {
1124                 return platform::exchange_ptr( pDest, val, order );
1125             }
1126             static T * atomic_exchange_explicit( T * * pDest, T * val, memory_order order ) CDS_NOEXCEPT
1127             {
1128                 return platform::exchange_ptr( pDest, val, order );
1129             }
1130             static T * atomic_exchange( T * volatile * pDest, T * val ) CDS_NOEXCEPT
1131             {
1132                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
1133             }
1134             static T * atomic_exchange( T * * pDest, T * val ) CDS_NOEXCEPT
1135             {
1136                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
1137             }
1138
1139             // cas
1140             static bool atomic_compare_exchange_weak_explicit( T * volatile * pDest, T * * expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
1141             {
1142                 assert( expected );
1143                 return platform::cas_ptr_weak( pDest, *expected, desired, mo_success, mo_fail );
1144             }
1145             static bool atomic_compare_exchange_weak_explicit( T * * pDest, T * * expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
1146             {
1147                 assert( expected );
1148                 return platform::cas_ptr_weak( pDest, *expected, desired, mo_success, mo_fail );
1149             }
1150             static bool atomic_compare_exchange_weak( T * volatile * pDest, T ** expected, T * desired ) CDS_NOEXCEPT
1151             {
1152                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1153             }
1154             static bool atomic_compare_exchange_weak( T ** pDest, T ** expected, T * desired ) CDS_NOEXCEPT
1155             {
1156                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1157             }
1158             static bool atomic_compare_exchange_strong_explicit( T * volatile * pDest, T ** expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
1159             {
1160                 assert( expected );
1161                 return platform::cas_ptr_strong( pDest, *expected, desired, mo_success, mo_fail );
1162             }
1163             static bool atomic_compare_exchange_strong_explicit( T ** pDest, T ** expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
1164             {
1165                 assert( expected );
1166                 return platform::cas_ptr_strong( pDest, *expected, desired, mo_success, mo_fail );
1167             }
1168             static bool atomic_compare_exchange_strong( T * volatile * pDest, T ** expected, T * desired ) CDS_NOEXCEPT
1169             {
1170                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1171             }
1172             static bool atomic_compare_exchange_strong( T ** pDest, T ** expected, T * desired ) CDS_NOEXCEPT
1173             {
1174                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1175             }
1176         };
1177
1178         template <typename T>
1179         struct atomic_pointer: public atomic_pointer_base<T>
1180         {
1181             typedef atomic_pointer_base<T> base_class;
1182             // fetch_add
1183             static T * atomic_fetch_add_explicit(T * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
1184             {
1185 #           ifdef CDS_ATOMIC_fetch_ptr_add_defined
1186                 platform::fetch_ptr_add( pDest, val, order );
1187 #           else
1188                 T * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1189                 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
1190                 return cur;
1191 #           endif
1192             }
1193             static T * atomic_fetch_add_explicit(T * * pDest, ptrdiff_t val , memory_order order) CDS_NOEXCEPT
1194             {
1195                 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1196             }
1197             static T * atomic_fetch_add( T * volatile * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1198             {
1199                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1200             }
1201             static T * atomic_fetch_add( T ** pDest, ptrdiff_t val ) CDS_NOEXCEPT
1202             {
1203                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1204             }
1205
1206             // fetch_sub
1207             static T * atomic_fetch_sub_explicit(T * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
1208             {
1209 #           ifdef CDS_ATOMIC_fetch_ptr_sub_defined
1210                 platform::fetch_ptr_sub( pDest, val, order );
1211 #           else
1212                 T * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1213                 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
1214                 return cur;
1215 #           endif
1216             }
1217             static T * atomic_fetch_sub_explicit(T ** pDest, ptrdiff_t val , memory_order order) CDS_NOEXCEPT
1218             {
1219                 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1220             }
1221             static T * atomic_fetch_sub( T volatile * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1222             {
1223                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1224             }
1225             static T * atomic_fetch_sub( T * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1226             {
1227                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1228             }
1229         };
1230
1231         template <>
1232         struct atomic_pointer<void>: public atomic_pointer_base<void>
1233         {
1234             typedef atomic_pointer_base<void>   base_class;
1235
1236             // fetch_add
1237             static void * atomic_fetch_add_explicit(void * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
1238             {
1239                 void * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1240                 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, reinterpret_cast<char *>(cur) + val, order, memory_order_relaxed ));
1241                 return cur;
1242             }
1243             static void * atomic_fetch_add_explicit(void * * pDest, ptrdiff_t val , memory_order order) CDS_NOEXCEPT
1244             {
1245                 return atomic_fetch_add_explicit( reinterpret_cast<void * volatile *>( pDest ), val, order );
1246             }
1247             static void * atomic_fetch_add( void * volatile * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1248             {
1249                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1250             }
1251             static void * atomic_fetch_add( void ** pDest, ptrdiff_t val ) CDS_NOEXCEPT
1252             {
1253                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1254             }
1255
1256             // fetch_sub
1257             static void * atomic_fetch_sub_explicit(void * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
1258             {
1259                 void * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1260                 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, reinterpret_cast<char *>(cur) - val, order, memory_order_relaxed ));
1261                 return cur;
1262             }
1263             static void * atomic_fetch_sub_explicit(void ** pDest, ptrdiff_t val , memory_order order) CDS_NOEXCEPT
1264             {
1265                 return atomic_fetch_sub_explicit( reinterpret_cast<void * volatile *>( pDest ), val, order );
1266             }
1267             static void * atomic_fetch_sub( void * volatile * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1268             {
1269                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1270             }
1271             static void * atomic_fetch_sub( void ** pDest, ptrdiff_t val ) CDS_NOEXCEPT
1272             {
1273                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1274             }
1275         };
1276
1277         template <typename T>
1278         struct atomic_integral
1279         {
1280         private:
1281             typename cds::details::aligned_type<T, sizeof(T)>::type volatile m_val;
1282             //T volatile  m_val;
1283             typedef atomic_integral_ops<T, sizeof(T)>   atomic_ops;
1284         public:
1285             typedef T   atomic_type;
1286         public:
1287             bool is_lock_free() const volatile CDS_NOEXCEPT
1288             {
1289                 return true;
1290             }
1291             bool is_lock_free() const CDS_NOEXCEPT
1292             {
1293                 return true;
1294             }
1295             void store(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1296             {
1297                 atomic_ops::atomic_store_explicit( &m_val, val, order );
1298             }
1299             void store(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1300             {
1301                 atomic_ops::atomic_store_explicit( &m_val, val, order );
1302             }
1303
1304             T load(memory_order order = memory_order_seq_cst) const volatile CDS_NOEXCEPT
1305             {
1306                 return atomic_ops::atomic_load_explicit( &m_val, order );
1307             }
1308             T load(memory_order order  = memory_order_seq_cst) const CDS_NOEXCEPT
1309             {
1310                 return atomic_ops::atomic_load_explicit( &m_val, order );
1311             }
1312
1313             operator T() const volatile CDS_NOEXCEPT
1314             {
1315                 return load();
1316             }
1317             operator T() const CDS_NOEXCEPT
1318             {
1319                 return load();
1320             }
1321
1322             T exchange(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1323             {
1324                 return atomic_ops::atomic_exchange_explicit( &m_val, val, order );
1325             }
1326             T exchange(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1327             {
1328                 return atomic_ops::atomic_exchange_explicit( &m_val, val, order );
1329             }
1330
1331             bool compare_exchange_weak(T& expected, T desired , memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1332             {
1333                 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_val, &expected, desired, success_order, failure_order );
1334             }
1335             bool compare_exchange_weak(T& expected, T desired , memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1336             {
1337                 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_val, &expected, desired, success_order, failure_order );
1338             }
1339             bool compare_exchange_strong(T& expected, T desired , memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1340             {
1341                 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_val, &expected, desired, success_order, failure_order );
1342             }
1343             bool compare_exchange_strong(T& expected, T desired , memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1344             {
1345                 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_val, &expected, desired, success_order, failure_order );
1346             }
1347             bool compare_exchange_weak(T& expected, T desired , memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1348             {
1349                 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1350             }
1351             bool compare_exchange_weak(T& expected, T desired , memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1352             {
1353                 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1354             }
1355             bool compare_exchange_strong(T& expected, T desired , memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1356             {
1357                 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1358             }
1359             bool compare_exchange_strong(T& expected, T desired , memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1360             {
1361                 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1362             }
1363
1364             T fetch_add(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1365             {
1366                 return atomic_ops::atomic_fetch_add_explicit( &m_val, val, order );
1367             }
1368             T fetch_add(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1369             {
1370                 return atomic_ops::atomic_fetch_add_explicit( &m_val, val, order );
1371             }
1372             T fetch_sub(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1373             {
1374                 return atomic_ops::atomic_fetch_sub_explicit( &m_val, val, order );
1375             }
1376             T fetch_sub(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1377             {
1378                 return atomic_ops::atomic_fetch_sub_explicit( &m_val, val, order );
1379             }
1380             T fetch_and(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1381             {
1382                 return atomic_ops::atomic_fetch_and_explicit( &m_val, val, order );
1383             }
1384             T fetch_and(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1385             {
1386                 return atomic_ops::atomic_fetch_and_explicit( &m_val, val, order );
1387             }
1388
1389             T fetch_or(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1390             {
1391                 return atomic_ops::atomic_fetch_or_explicit( &m_val, val, order );
1392             }
1393             T fetch_or(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1394             {
1395                 return atomic_ops::atomic_fetch_or_explicit( &m_val, val, order );
1396             }
1397             T fetch_xor(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1398             {
1399                 return atomic_ops::atomic_fetch_xor_explicit( &m_val, val, order );
1400             }
1401             T fetch_xor(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1402             {
1403                 return atomic_ops::atomic_fetch_xor_explicit( &m_val, val, order );
1404             }
1405
1406             atomic_integral() = default;
1407             CDS_CONSTEXPR atomic_integral(T val) CDS_NOEXCEPT
1408                 : m_val(val)
1409                 {}
1410
1411             atomic_integral(const atomic_integral&) = delete;
1412             atomic_integral& operator=(const atomic_integral&) = delete;
1413             atomic_integral& operator=(const atomic_integral&) volatile = delete;
1414
1415             T operator=(T val) volatile CDS_NOEXCEPT
1416             {
1417                 store(val);
1418                 return val;
1419             }
1420             T operator=(T val) CDS_NOEXCEPT
1421             {
1422                 store(val);
1423                 return val;
1424             }
1425
1426             // Post inc/dec
1427             T operator++(int) volatile CDS_NOEXCEPT
1428             {
1429                 return fetch_add( 1 );
1430             }
1431             T operator++(int) CDS_NOEXCEPT
1432             {
1433                 return fetch_add( 1 );
1434             }
1435             T operator--(int) volatile CDS_NOEXCEPT
1436             {
1437                 return fetch_sub( 1 );
1438             }
1439             T operator--(int) CDS_NOEXCEPT
1440             {
1441                 return fetch_sub( 1 );
1442             }
1443
1444             // Pre inc/dec
1445             T operator++() volatile CDS_NOEXCEPT
1446             {
1447                 return fetch_add( 1 ) + 1;
1448             }
1449             T operator++() CDS_NOEXCEPT
1450             {
1451                 return fetch_add( 1 ) + 1;
1452             }
1453             T operator--() volatile CDS_NOEXCEPT
1454             {
1455                 return fetch_sub( 1 ) - 1;
1456             }
1457             T operator--() CDS_NOEXCEPT
1458             {
1459                 return fetch_sub( 1 ) - 1;
1460             }
1461
1462             // op=
1463             T operator+=(T val) volatile CDS_NOEXCEPT
1464             {
1465                 return fetch_add( val ) + val;
1466             }
1467             T operator+=(T val) CDS_NOEXCEPT
1468             {
1469                 return fetch_add( val ) + val;
1470             }
1471             T operator-=(T val) volatile CDS_NOEXCEPT
1472             {
1473                 return fetch_sub( val ) - val;
1474             }
1475             T operator-=(T val) CDS_NOEXCEPT
1476             {
1477                 return fetch_sub( val ) - val;
1478             }
1479             T operator&=(T val) volatile CDS_NOEXCEPT
1480             {
1481                 return fetch_and( val ) & val;
1482             }
1483             T operator&=(T val) CDS_NOEXCEPT
1484             {
1485                 return fetch_and( val ) & val;
1486             }
1487             T operator|=(T val) volatile CDS_NOEXCEPT
1488             {
1489                 return fetch_or( val ) | val;
1490             }
1491             T operator|=(T val) CDS_NOEXCEPT
1492             {
1493                 return fetch_or( val ) | val;
1494             }
1495             T operator^=(T val) volatile CDS_NOEXCEPT
1496             {
1497                 return fetch_xor( val ) ^ val;
1498             }
1499             T operator^=(T val) CDS_NOEXCEPT
1500             {
1501                 return fetch_xor( val ) ^ val;
1502             }
1503         };
1504
1505         template <typename Type>
1506         struct select_primary_type {
1507             typedef typename details::primary_type<sizeof(Type)>::type type;
1508         };
1509         template <>
1510         struct select_primary_type<bool> {
1511             typedef bool type;
1512         };
1513
1514     }   // namespace details
1515
1516     template <class T>
1517     struct atomic
1518     {
1519     private:
1520         typedef details::atomic_generic_ops<T, sizeof(T), typename details::select_primary_type<T>::type >  atomic_ops;
1521
1522         T volatile m_data;
1523     public:
1524         bool is_lock_free() const volatile CDS_NOEXCEPT
1525         {
1526             return true;
1527         }
1528         bool is_lock_free() const CDS_NOEXCEPT
1529         {
1530             return true;
1531         }
1532
1533         void store(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1534         {
1535             atomic_ops::atomic_store_explicit( &m_data, val, order );
1536         }
1537         void store(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1538         {
1539             atomic_ops::atomic_store_explicit( &m_data, val, order );
1540         }
1541
1542         T load(memory_order order = memory_order_seq_cst) const volatile CDS_NOEXCEPT
1543         {
1544             return atomic_ops::atomic_load_explicit( &m_data, order );
1545         }
1546         T load(memory_order order = memory_order_seq_cst) const CDS_NOEXCEPT
1547         {
1548            return atomic_ops::atomic_load_explicit( &m_data, order );
1549         }
1550
1551         operator T() const volatile CDS_NOEXCEPT
1552         {
1553             return load();
1554         }
1555         operator T() const CDS_NOEXCEPT
1556         {
1557             return load();
1558         }
1559
1560         T exchange(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1561         {
1562             return atomic_ops::atomic_exchange_explicit( &m_data, val, order );
1563         }
1564         T exchange(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1565         {
1566             return atomic_ops::atomic_exchange_explicit( &m_data, val, order );
1567         }
1568
1569         bool compare_exchange_weak(T& expected, T desired, memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1570         {
1571             return atomic_ops::atomic_compare_exchange_weak_explicit( &m_data, &expected, desired, success_order, failure_order );
1572         }
1573         bool compare_exchange_weak(T& expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1574         {
1575             return atomic_ops::atomic_compare_exchange_weak_explicit( &m_data, &expected, desired, success_order, failure_order );
1576         }
1577         bool compare_exchange_strong(T& expected, T desired, memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1578         {
1579             return atomic_ops::atomic_compare_exchange_strong_explicit( &m_data, &expected, desired, success_order, failure_order );
1580         }
1581         bool compare_exchange_strong(T& expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1582         {
1583             return atomic_ops::atomic_compare_exchange_strong_explicit( &m_data, &expected, desired, success_order, failure_order );
1584         }
1585         bool compare_exchange_weak(T& expected, T desired, memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1586         {
1587             return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1588         }
1589         bool compare_exchange_weak(T& expected, T desired, memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1590         {
1591             return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1592         }
1593         bool compare_exchange_strong(T& expected, T desired, memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1594         {
1595             return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1596         }
1597         bool compare_exchange_strong(T& expected, T desired, memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1598         {
1599             return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1600         }
1601
1602         atomic() = default;
1603         CDS_CONSTEXPR atomic(T val)
1604             : m_data( val )
1605             {}
1606
1607         atomic(const atomic&) = delete;
1608         atomic& operator=(const atomic&) = delete;
1609         atomic& operator=(const atomic&) volatile = delete;
1610
1611         T operator=(T val) volatile CDS_NOEXCEPT
1612         {
1613             store( val );
1614             return val;
1615         }
1616         T operator=(T val) CDS_NOEXCEPT
1617         {
1618             store( val );
1619             return val;
1620         }
1621     };
1622
1623 #   define CDS_DECLARE_ATOMIC_INTEGRAL( _type ) \
1624     template <> \
1625     struct atomic<_type>: public details::atomic_integral<_type> \
1626     { \
1627     private: \
1628         typedef details::atomic_integral<_type>   base_class  ; \
1629     public: \
1630         atomic() = default; \
1631         atomic(_type val) CDS_NOEXCEPT : base_class(val) {} \
1632         atomic(const atomic&) = delete; \
1633         atomic& operator=(const atomic&) = delete; \
1634         atomic& operator=(const atomic&) volatile = delete; \
1635         _type operator=(_type val) volatile CDS_NOEXCEPT { return base_class::operator=(val); } \
1636         _type operator=(_type val) CDS_NOEXCEPT { return base_class::operator=(val); } \
1637     };
1638
1639     CDS_DECLARE_ATOMIC_INTEGRAL(char)
1640     CDS_DECLARE_ATOMIC_INTEGRAL(signed char)
1641     CDS_DECLARE_ATOMIC_INTEGRAL(unsigned char)
1642     CDS_DECLARE_ATOMIC_INTEGRAL(short)
1643     CDS_DECLARE_ATOMIC_INTEGRAL(unsigned short)
1644     CDS_DECLARE_ATOMIC_INTEGRAL(int)
1645     CDS_DECLARE_ATOMIC_INTEGRAL(unsigned int)
1646     CDS_DECLARE_ATOMIC_INTEGRAL(long)
1647     CDS_DECLARE_ATOMIC_INTEGRAL(unsigned long)
1648     CDS_DECLARE_ATOMIC_INTEGRAL(long long)
1649     CDS_DECLARE_ATOMIC_INTEGRAL(unsigned long long)
1650 //#if CDS_COMPILER == CDS_COMPILER_GCC && CDS_COMPILER_VERSION >= 40400
1651 //    CDS_DECLARE_ATOMIC_INTEGRAL(char16_t)
1652 //    CDS_DECLARE_ATOMIC_INTEGRAL(char32_t)
1653 //#endif
1654 //    CDS_DECLARE_ATOMIC_INTEGRAL(wchar_t)
1655
1656 #   undef CDS_DECLARE_ATOMIC_INTEGRAL
1657
1658
1659     template <typename T>
1660     class atomic<T *>
1661     {
1662     private:
1663         T * volatile m_ptr;
1664         typedef details::atomic_pointer<T>  atomic_ops;
1665     public:
1666         bool is_lock_free() const volatile CDS_NOEXCEPT
1667         {
1668             return true;
1669         }
1670         bool is_lock_free() const CDS_NOEXCEPT
1671         {
1672             return true;
1673         }
1674
1675         void store(T * val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1676         {
1677             atomic_ops::atomic_store_explicit( &m_ptr, val, order );
1678         }
1679         void store(T * val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1680         {
1681             atomic_ops::atomic_store_explicit( &m_ptr, val, order );
1682         }
1683
1684         T * load(memory_order order = memory_order_seq_cst) const volatile CDS_NOEXCEPT
1685         {
1686             return atomic_ops::atomic_load_explicit( &m_ptr, order );
1687         }
1688         T * load(memory_order order = memory_order_seq_cst) const CDS_NOEXCEPT
1689         {
1690             return atomic_ops::atomic_load_explicit( &m_ptr, order );
1691         }
1692
1693         operator T *() const volatile CDS_NOEXCEPT
1694         {
1695             return load();
1696         }
1697         operator T *() const CDS_NOEXCEPT
1698         {
1699             return load();
1700         }
1701
1702         T * exchange(T * val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1703         {
1704             return atomic_ops::atomic_exchange_explicit( &m_ptr, val, order );
1705         }
1706         T * exchange(T * val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1707         {
1708             return atomic_ops::atomic_exchange_explicit( &m_ptr, val, order );
1709         }
1710
1711         bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1712         {
1713             return atomic_ops::atomic_compare_exchange_weak_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1714         }
1715         bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1716         {
1717             return atomic_ops::atomic_compare_exchange_weak_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1718         }
1719         bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1720         {
1721             return atomic_ops::atomic_compare_exchange_strong_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1722         }
1723         bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1724         {
1725             return atomic_ops::atomic_compare_exchange_strong_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1726         }
1727         bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1728         {
1729             return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1730         }
1731         bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1732         {
1733             return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1734         }
1735         bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1736         {
1737             return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1738         }
1739         bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1740         {
1741             return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1742         }
1743
1744         T * fetch_add(ptrdiff_t offset, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1745         {
1746             return atomic_ops::atomic_fetch_add_explicit( &m_ptr, offset, order );
1747         }
1748         T * fetch_add(ptrdiff_t offset, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1749         {
1750             return atomic_ops::atomic_fetch_add_explicit( &m_ptr, offset, order );
1751         }
1752
1753         T * fetch_sub(ptrdiff_t offset, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1754         {
1755             return atomic_ops::atomic_fetch_sub_explicit( &m_ptr, offset, order );
1756         }
1757         T * fetch_sub(ptrdiff_t offset, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1758         {
1759             return atomic_ops::atomic_fetch_sub_explicit( &m_ptr, offset, order );
1760         }
1761
1762         atomic() = default;
1763         CDS_CONSTEXPR atomic(T * val) CDS_NOEXCEPT
1764             : m_ptr( val )
1765         {}
1766
1767         atomic(const atomic&) = delete;
1768         atomic& operator=(const atomic&) = delete;
1769         atomic& operator=(const atomic&) volatile = delete;
1770
1771         T * operator=(T * val) volatile CDS_NOEXCEPT
1772         {
1773             store( val );
1774             return val;
1775         }
1776         T * operator=(T * val) CDS_NOEXCEPT
1777         {
1778             store( val );
1779             return val;
1780         }
1781     };
1782
1783     // Atomic typedefs
1784     typedef atomic<bool>            atomic_bool;
1785     typedef atomic<char>            atomic_char;
1786     typedef atomic<signed char>     atomic_schar;
1787     typedef atomic<unsigned char>   atomic_uchar;
1788     typedef atomic<short>           atomic_short;
1789     typedef atomic<unsigned short>  atomic_ushort;
1790     typedef atomic<int>             atomic_int;
1791     typedef atomic<unsigned int>    atomic_uint;
1792     typedef atomic<long>            atomic_long;
1793     typedef atomic<unsigned long>   atomic_ulong;
1794     typedef atomic<long long>       atomic_llong;
1795     typedef atomic<unsigned long long> atomic_ullong;
1796 #if ( CDS_COMPILER == CDS_COMPILER_GCC && CDS_COMPILER_VERSION >= 40400 ) || CDS_COMPILER == CDS_COMPILER_CLANG
1797     typedef atomic<char16_t>        atomic_char16_t;
1798     typedef atomic<char32_t>        atomic_char32_t;
1799 #endif
1800     typedef atomic<wchar_t>         atomic_wchar_t;
1801
1802
1803     typedef atomic<std::int_least8_t>    atomic_int_least8_t;
1804     typedef atomic<std::uint_least8_t>   atomic_uint_least8_t;
1805     typedef atomic<std::int_least16_t>   atomic_int_least16_t;
1806     typedef atomic<std::uint_least16_t>  atomic_uint_least16_t;
1807     typedef atomic<std::int_least32_t>   atomic_int_least32_t;
1808     typedef atomic<std::uint_least32_t>  atomic_uint_least32_t;
1809     typedef atomic<std::int_least64_t>   atomic_int_least64_t;
1810     typedef atomic<std::uint_least64_t>  atomic_uint_least64_t;
1811     typedef atomic<std::int_fast8_t>     atomic_int_fast8_t;
1812     typedef atomic<std::uint_fast8_t>    atomic_uint_fast8_t;
1813     typedef atomic<std::int_fast16_t>    atomic_int_fast16_t;
1814     typedef atomic<std::uint_fast16_t>   atomic_uint_fast16_t;
1815     typedef atomic<std::int_fast32_t>    atomic_int_fast32_t;
1816     typedef atomic<std::uint_fast32_t>   atomic_uint_fast32_t;
1817     typedef atomic<std::int_fast64_t>    atomic_int_fast64_t;
1818     typedef atomic<std::uint_fast64_t>   atomic_uint_fast64_t;
1819     typedef atomic<intptr_t>             atomic_intptr_t;
1820     typedef atomic<uintptr_t>            atomic_uintptr_t;
1821     typedef atomic<size_t>               atomic_size_t;
1822     typedef atomic<ptrdiff_t>            atomic_ptrdiff_t;
1823     typedef atomic<std::intmax_t>        atomic_intmax_t;
1824     typedef atomic<std::uintmax_t>       atomic_uintmax_t;
1825
1826     template <class T>
1827     static inline bool atomic_is_lock_free(const volatile atomic<T> * p) CDS_NOEXCEPT
1828     {
1829         return p->is_lock_free();
1830     }
1831
1832     template <class T>
1833     static inline bool atomic_is_lock_free(const atomic<T> * p ) CDS_NOEXCEPT
1834     {
1835         return p->is_lock_free();
1836     }
1837
1838     /*
1839     template <class T>
1840     static inline void atomic_init(volatile atomic<T> * p, T val) CDS_NOEXCEPT
1841     {
1842         p->init( val );
1843     }
1844
1845     template <class T>
1846     static inline void atomic_init( atomic<T> * p, T val) CDS_NOEXCEPT
1847     {
1848         p->init( val );
1849     }
1850     */
1851
1852     template <class T>
1853     static inline void atomic_store(volatile atomic<T>* p, T val) CDS_NOEXCEPT
1854     {
1855         p->store(val);
1856     }
1857     template <class T>
1858     static inline void atomic_store(atomic<T>* p, T val) CDS_NOEXCEPT
1859     {
1860         p->store( val );
1861     }
1862
1863     template <class T>
1864     static inline void atomic_store_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1865     {
1866         p->store( val, order );
1867     }
1868     template <class T>
1869     static inline void atomic_store_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1870     {
1871         p->store( val, order );
1872     }
1873
1874     template <class T>
1875     static inline T atomic_load(const volatile atomic<T>* p) CDS_NOEXCEPT
1876     {
1877         return p->load();
1878     }
1879     template <class T>
1880     static inline T atomic_load(const atomic<T>* p) CDS_NOEXCEPT
1881     {
1882         return p->load();
1883     }
1884
1885     template <class T>
1886     static inline T atomic_load_explicit(const volatile atomic<T>* p, memory_order order) CDS_NOEXCEPT
1887     {
1888         return p->load( order );
1889     }
1890     template <class T>
1891     static inline T atomic_load_explicit(const atomic<T>* p, memory_order order) CDS_NOEXCEPT
1892     {
1893         return p->load( order );
1894     }
1895
1896     template <class T>
1897     static inline T atomic_exchange(volatile atomic<T>* p, T val) CDS_NOEXCEPT
1898     {
1899         return p->exchange( val );
1900     }
1901     template <class T>
1902     static inline T atomic_exchange(atomic<T>* p, T val ) CDS_NOEXCEPT
1903     {
1904         return p->exchange( val );
1905     }
1906
1907     template <class T>
1908     static inline T atomic_exchange_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1909     {
1910         return p->exchange( val, order );
1911     }
1912     template <class T>
1913     static inline T atomic_exchange_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1914     {
1915         return p->exchange( val, order );
1916     }
1917
1918     template <class T>
1919     static inline bool atomic_compare_exchange_weak(volatile atomic<T>* p, T* expected, T desired) CDS_NOEXCEPT
1920     {
1921         return p->compare_exchange_weak( *expected, desired );
1922     }
1923     template <class T>
1924     static inline bool atomic_compare_exchange_weak(atomic<T>* p, T* expected, T desired) CDS_NOEXCEPT
1925     {
1926         return p->compare_exchange_weak( *expected, desired );
1927     }
1928
1929     template <class T>
1930     static inline bool atomic_compare_exchange_strong(volatile atomic<T>* p, T* expected, T desired) CDS_NOEXCEPT
1931     {
1932         return p->compare_exchange_strong( *expected, desired );
1933     }
1934     template <class T>
1935     static inline bool atomic_compare_exchange_strong(atomic<T>* p, T* expected, T desired) CDS_NOEXCEPT
1936     {
1937         return p->compare_exchange_strong( *expected, desired );
1938     }
1939
1940     template <class T>
1941     static inline bool atomic_compare_exchange_weak_explicit(volatile atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1942     {
1943         return p->compare_exchange_weak( *expected, desired, success_order, failure_order );
1944     }
1945     template <class T>
1946     static inline bool atomic_compare_exchange_weak_explicit(atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1947     {
1948         return p->compare_exchange_weak( *expected, desired, success_order, failure_order );
1949     }
1950
1951     template <class T>
1952     static inline bool atomic_compare_exchange_strong_explicit(volatile atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1953     {
1954         return p->compare_exchange_strong( *expected, desired, success_order, failure_order );
1955     }
1956     template <class T>
1957     static inline bool atomic_compare_exchange_strong_explicit(atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1958     {
1959         return p->compare_exchange_strong( *expected, desired, success_order, failure_order );
1960     }
1961
1962     template <class T>
1963     static inline T atomic_fetch_add(volatile atomic<T>* p, T val) CDS_NOEXCEPT
1964     {
1965         return p->fetch_add( val );
1966     }
1967     template <class T>
1968     static inline T atomic_fetch_add(atomic<T>* p, T val) CDS_NOEXCEPT
1969     {
1970         return p->fetch_add( val );
1971     }
1972     template <class T>
1973     static inline T * atomic_fetch_add(volatile atomic<T *>* p, ptrdiff_t offset) CDS_NOEXCEPT
1974     {
1975         return p->fetch_add( offset );
1976     }
1977     template <class T>
1978     static inline T * atomic_fetch_add(atomic<T *>* p, ptrdiff_t offset) CDS_NOEXCEPT
1979     {
1980         return p->fetch_add( offset );
1981     }
1982
1983     template <class T>
1984     static inline T atomic_fetch_add_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1985     {
1986         return p->fetch_add( val, order );
1987     }
1988     template <class T>
1989     static inline T atomic_fetch_add_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1990     {
1991         return p->fetch_add( val, order );
1992     }
1993     template <class T>
1994     static inline T * atomic_fetch_add_explicit(volatile atomic<T *>* p, ptrdiff_t offset, memory_order order) CDS_NOEXCEPT
1995     {
1996         return p->fetch_add( offset, order );
1997     }
1998     template <class T>
1999     static inline T * atomic_fetch_add_explicit(atomic<T *>* p, ptrdiff_t offset, memory_order order) CDS_NOEXCEPT
2000     {
2001         return p->fetch_add( offset, order );
2002     }
2003
2004     template <class T>
2005     static inline T atomic_fetch_sub(volatile atomic<T>* p, T val) CDS_NOEXCEPT
2006     {
2007         return p->fetch_sub( val );
2008     }
2009     template <class T>
2010     static inline T atomic_fetch_sub(atomic<T>* p, T val) CDS_NOEXCEPT
2011     {
2012         return p->fetch_sub( val );
2013     }
2014     template <class T>
2015     static inline T * atomic_fetch_sub(volatile atomic<T *>* p, ptrdiff_t offset) CDS_NOEXCEPT
2016     {
2017         return p->fetch_sub( offset );
2018     }
2019     template <class T>
2020     static inline T * atomic_fetch_sub(atomic<T *>* p, ptrdiff_t offset) CDS_NOEXCEPT
2021     {
2022         return p->fetch_sub( offset );
2023     }
2024
2025     template <class T>
2026     static inline T atomic_fetch_sub_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2027     {
2028         return p->fetch_sub( val, order );
2029     }
2030     template <class T>
2031     static inline T atomic_fetch_sub_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2032     {
2033         return p->fetch_sub( val, order );
2034     }
2035     template <class T>
2036     static inline T * atomic_fetch_sub_explicit(volatile atomic<T *>* p, ptrdiff_t offset, memory_order order) CDS_NOEXCEPT
2037     {
2038         return p->fetch_sub( offset, order );
2039     }
2040     template <class T>
2041     static inline T * atomic_fetch_sub_explicit(atomic<T *>* p, ptrdiff_t offset, memory_order order) CDS_NOEXCEPT
2042     {
2043         return p->fetch_sub( offset, order );
2044     }
2045
2046     template <class T>
2047     static inline T atomic_fetch_and(volatile atomic<T>* p, T val) CDS_NOEXCEPT
2048     {
2049         return p->fetch_and( val );
2050     }
2051     template <class T>
2052     static inline T atomic_fetch_and(atomic<T>* p, T val) CDS_NOEXCEPT
2053     {
2054         return p->fetch_and( val );
2055     }
2056
2057     template <class T>
2058     static inline T atomic_fetch_and_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2059     {
2060         return p->fetch_and( val, order );
2061     }
2062     template <class T>
2063     static inline T atomic_fetch_and_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2064     {
2065         return p->fetch_and( val, order );
2066     }
2067
2068     template <class T>
2069     static inline T atomic_fetch_or(volatile atomic<T>* p, T val) CDS_NOEXCEPT
2070     {
2071         return p->fetch_or( val );
2072     }
2073     template <class T>
2074     static inline T atomic_fetch_or(atomic<T>* p, T val) CDS_NOEXCEPT
2075     {
2076         return p->fetch_or( val );
2077     }
2078
2079     template <class T>
2080     static inline T atomic_fetch_or_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2081     {
2082         return p->fetch_or( val, order );
2083     }
2084     template <class T>
2085     static inline T atomic_fetch_or_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2086     {
2087         return p->fetch_or( val, order );
2088     }
2089
2090     template <class T>
2091     static inline T atomic_fetch_xor(volatile atomic<T>* p, T val) CDS_NOEXCEPT
2092     {
2093         return p->fetch_xor( val );
2094     }
2095     template <class T>
2096     static inline T atomic_fetch_xor(atomic<T>* p, T val) CDS_NOEXCEPT
2097     {
2098         return p->fetch_xor( val );
2099     }
2100
2101     template <class T>
2102     static inline T atomic_fetch_xor_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2103     {
2104         return p->fetch_xor( val, order );
2105     }
2106     template <class T>
2107     static inline T atomic_fetch_xor_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2108     {
2109         return p->fetch_xor( val, order );
2110     }
2111
2112     // Atomic flag type
2113     typedef struct atomic_flag
2114     {
2115         void clear( memory_order order = memory_order_seq_cst ) volatile CDS_NOEXCEPT
2116         {
2117             assert( order != memory_order_acquire
2118                 && order != memory_order_acq_rel
2119                 && order != memory_order_consume
2120                 );
2121             platform::atomic_flag_clear( &m_Flag, order );
2122         }
2123         void clear( memory_order order = memory_order_seq_cst ) CDS_NOEXCEPT
2124         {
2125             assert( order != memory_order_acquire
2126                 && order != memory_order_acq_rel
2127                 && order != memory_order_consume
2128                 );
2129             platform::atomic_flag_clear( &m_Flag, order );
2130         }
2131
2132         bool test_and_set( memory_order order = memory_order_seq_cst ) volatile CDS_NOEXCEPT
2133         {
2134             return platform::atomic_flag_tas( &m_Flag, order );
2135         }
2136         bool test_and_set( memory_order order = memory_order_seq_cst ) CDS_NOEXCEPT
2137         {
2138             return platform::atomic_flag_tas( &m_Flag, order );
2139         }
2140
2141         atomic_flag() = default;
2142
2143         atomic_flag(const atomic_flag&) = delete;
2144         atomic_flag& operator=(const atomic_flag&) = delete;
2145         atomic_flag& operator=(const atomic_flag&) volatile = delete;
2146
2147         platform::atomic_flag_type volatile m_Flag;
2148     } atomic_flag;
2149
2150     static inline bool atomic_flag_test_and_set(volatile atomic_flag* p) CDS_NOEXCEPT
2151     {
2152         return p->test_and_set();
2153     }
2154     static inline bool atomic_flag_test_and_set(atomic_flag * p) CDS_NOEXCEPT
2155     {
2156         return p->test_and_set();
2157     }
2158     static inline bool atomic_flag_test_and_set_explicit(volatile atomic_flag* p, memory_order order) CDS_NOEXCEPT
2159     {
2160         return p->test_and_set( order );
2161     }
2162     static inline bool atomic_flag_test_and_set_explicit(atomic_flag* p, memory_order order) CDS_NOEXCEPT
2163     {
2164         return p->test_and_set( order );
2165     }
2166     static inline void atomic_flag_clear(volatile atomic_flag* p) CDS_NOEXCEPT
2167     {
2168         return p->clear();
2169     }
2170     static inline void atomic_flag_clear(atomic_flag* p) CDS_NOEXCEPT
2171     {
2172         return p->clear();
2173     }
2174     static inline void atomic_flag_clear_explicit(volatile atomic_flag* p, memory_order order) CDS_NOEXCEPT
2175     {
2176         return p->clear( order );
2177     }
2178     static inline void atomic_flag_clear_explicit(atomic_flag* p, memory_order order) CDS_NOEXCEPT
2179     {
2180         return p->clear( order );
2181     }
2182
2183     // Fences
2184     static inline void atomic_thread_fence(memory_order order) CDS_NOEXCEPT
2185     {
2186         platform::thread_fence( order );
2187         CDS_COMPILER_RW_BARRIER;
2188     }
2189     static inline void atomic_signal_fence(memory_order order) CDS_NOEXCEPT
2190     {
2191         platform::signal_fence( order );
2192     }
2193
2194 }}  // namespace cds::cxx11_atomic
2195
2196 //@endcond
2197 #endif // #ifndef CDSLIB_COMPILER_CXX11_ATOMIC_H