cds/compiler/cxx11_atomic.h

   1 //$$CDS-header$$
   2
   3 #ifndef __CDS_COMPILER_CXX11_ATOMIC_H
   4 #define __CDS_COMPILER_CXX11_ATOMIC_H
   5 //@cond
   6
   7 #include <cds/details/defs.h>
   8 #include <cds/details/aligned_type.h>
   9
  10 namespace cds { namespace cxx11_atomics {
  11     typedef enum memory_order {
  12         memory_order_relaxed,
  13         memory_order_consume,
  14         memory_order_acquire,
  15         memory_order_release,
  16         memory_order_acq_rel,
  17         memory_order_seq_cst
  18     } memory_order;
  19
  20 }}  // namespace cds::cxx11_atomics
  21
  22
  23 #if CDS_COMPILER == CDS_COMPILER_MSVC || (CDS_COMPILER == CDS_COMPILER_INTEL && CDS_OS_INTERFACE == CDS_OSI_WINDOWS)
  24 #   if CDS_PROCESSOR_ARCH == CDS_PROCESSOR_X86
  25 #       include <cds/compiler/vc/x86/cxx11_atomic.h>
  26 #   elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_AMD64
  27 #       include <cds/compiler/vc/amd64/cxx11_atomic.h>
  28 #   else
  29 #       error "MS VC++ compiler: unsupported processor architecture"
  30 #   endif
  31 #elif CDS_COMPILER == CDS_COMPILER_GCC || CDS_COMPILER == CDS_COMPILER_CLANG || CDS_COMPILER == CDS_COMPILER_INTEL
  32 #   if CDS_PROCESSOR_ARCH == CDS_PROCESSOR_X86
  33 #       include <cds/compiler/gcc/x86/cxx11_atomic.h>
  34 #   elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_AMD64
  35 #       include <cds/compiler/gcc/amd64/cxx11_atomic.h>
  36 #   elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_IA64
  37 #       include <cds/compiler/gcc/ia64/cxx11_atomic.h>
  38 #   elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_SPARC
  39 #       include <cds/compiler/gcc/sparc/cxx11_atomic.h>
  40 #   elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_PPC64
  41 #       include <cds/compiler/gcc/ppc64/cxx11_atomic.h>
  42 //#   elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_ARM7
  43 //#       include <cds/compiler/gcc/arm7/cxx11_atomic.h>
  44 #   else
  45 #       error "GCC compiler: unsupported processor architecture. Try to use native C++11 atomic or boost.atomic"
  46 #   endif
  47 #else
  48 #   error "Undefined compiler"
  49 #endif
  50
  51 // In C++11, make_unsigned is declared in <type_traits>
  52 #include <boost/type_traits/make_unsigned.hpp>  // for make_unsigned
  53
  54 namespace cds { namespace cxx11_atomics {
  55
  56     // forward declarations
  57     template <class T>
  58     struct atomic;
  59
  60     namespace details {
  61
  62         template <typename T, size_t Size, typename Primary = T >
  63         struct atomic_generic_ops;
  64
  65         template <typename T, size_t Size>
  66         struct atomic_integral_ops;
  67
  68         template <size_t TypeSize>
  69         struct primary_type;
  70
  71         template <>
  72         struct primary_type<1>
  73         {
  74             typedef cds::uint8_t type;
  75         };
  76         template <>
  77         struct primary_type<2>
  78         {
  79             typedef cds::uint16_t type;
  80         };
  81         template <>
  82         struct primary_type<4>
  83         {
  84             typedef cds::uint32_t type;
  85         };
  86         template <>
  87         struct primary_type<8>
  88         {
  89             typedef cds::uint64_t type;
  90         };
  91
  92         template <typename T, typename Primary>
  93         struct make_atomic_primary
  94         {
  95             typedef T       source_type;
  96             typedef Primary primary_type;
  97
  98             static primary_type volatile * ptr( source_type volatile * p ) CDS_NOEXCEPT
  99             {
 100                 return reinterpret_cast<primary_type volatile *>(p);
 101             }
 102             static primary_type const volatile * ptr( source_type const volatile * p ) CDS_NOEXCEPT
 103             {
 104                 return reinterpret_cast<primary_type const volatile *>(p);
 105             }
 106
 107             static primary_type val( source_type v ) CDS_NOEXCEPT
 108             {
 109                 return *reinterpret_cast<primary_type*>(&v);
 110             }
 111
 112             static primary_type& ref( source_type& v ) CDS_NOEXCEPT
 113             {
 114                 return reinterpret_cast<primary_type&>(v);
 115             }
 116
 117             static primary_type const& ref( source_type const& v ) CDS_NOEXCEPT
 118             {
 119                 return reinterpret_cast<primary_type const&>(v);
 120             }
 121
 122             static source_type ret( primary_type r ) CDS_NOEXCEPT
 123             {
 124                 return *reinterpret_cast<source_type *>(&r);
 125             }
 126         };
 127
 128         template <typename T>
 129         struct make_atomic_primary<T, T>
 130         {
 131             typedef T source_type;
 132             typedef T primary_type;
 133
 134             static primary_type volatile * ptr( source_type volatile * p ) CDS_NOEXCEPT
 135             {
 136                 return p;
 137             }
 138             static primary_type const volatile * ptr( source_type const volatile * p ) CDS_NOEXCEPT
 139             {
 140                 return p;
 141             }
 142
 143             static primary_type val( source_type v ) CDS_NOEXCEPT
 144             {
 145                 return v;
 146             }
 147
 148             static primary_type& ref( source_type& v ) CDS_NOEXCEPT
 149             {
 150                 return v;
 151             }
 152
 153             static source_type ret( primary_type r ) CDS_NOEXCEPT
 154             {
 155                 return r;
 156             }
 157         };
 158
 159         template <typename T>
 160         struct atomic_integral_bitwise_ops
 161         {
 162         public:
 163             typedef typename boost::make_unsigned<T>::type unsigned_type;
 164             typedef atomic_generic_ops<unsigned_type, sizeof(unsigned_type)> atomic_ops;
 165
 166             static T fetch_and(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 167             {
 168                 unsigned_type cur = atomic_ops::atomic_load_explicit( reinterpret_cast<unsigned_type volatile *>(pDest), memory_order_relaxed );
 169                 do {} while ( !atomic_ops::atomic_compare_exchange_weak_explicit(
 170                     reinterpret_cast<unsigned_type volatile *>(pDest), &cur, cur & unsigned_type(val), order, memory_order_relaxed ));
 171                 return T(cur);
 172             }
 173
 174             static T fetch_or(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 175             {
 176                 unsigned_type cur = atomic_ops::atomic_load_explicit( reinterpret_cast<unsigned_type volatile *>(pDest), memory_order_relaxed );
 177                 do {} while ( !atomic_ops::atomic_compare_exchange_weak_explicit(
 178                     reinterpret_cast<unsigned_type volatile *>(pDest), &cur, cur | unsigned_type(val), order, memory_order_relaxed ));
 179                 return T(cur);
 180             }
 181
 182             static T fetch_xor(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 183             {
 184                 unsigned_type cur = atomic_ops::atomic_load_explicit( reinterpret_cast<unsigned_type volatile *>(pDest), memory_order_relaxed );
 185                 do {} while ( !atomic_ops::atomic_compare_exchange_weak_explicit(
 186                     reinterpret_cast<unsigned_type volatile *>(pDest), &cur, cur ^ unsigned_type(val), order, memory_order_relaxed ));
 187                 return T(cur);
 188             }
 189         };
 190
 191
 192         // 8-bit atomic operations
 193
 194         template <typename T, typename Primary>
 195         struct atomic_generic_ops< T, 1, Primary >
 196         {
 197             typedef make_atomic_primary<T, Primary> primary;
 198
 199             // store
 200             static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
 201             {
 202                 platform::store8( primary::ptr(pDest), primary::val(v), order );
 203             }
 204             static void atomic_store_explicit( T * pDest, T v, memory_order order ) CDS_NOEXCEPT
 205             {
 206                 platform::store8( primary::ptr(pDest), primary::val(v), order );
 207             }
 208             static void atomic_store( T volatile * pDest, T v ) CDS_NOEXCEPT
 209             {
 210                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 211             }
 212             static void atomic_store( T * pDest, T v ) CDS_NOEXCEPT
 213             {
 214                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 215             }
 216
 217             // load
 218             static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
 219             {
 220                 return primary::ret( platform::load8( primary::ptr(pSrc), order ));
 221             }
 222             static T atomic_load_explicit( T const * pSrc, memory_order order ) CDS_NOEXCEPT
 223             {
 224                 return primary::ret( platform::load8( primary::ptr(pSrc), order ));
 225             }
 226             static T atomic_load( T volatile const * pSrc ) CDS_NOEXCEPT
 227             {
 228                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 229             }
 230             static T atomic_load( T const * pSrc ) CDS_NOEXCEPT
 231             {
 232                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 233             }
 234
 235             // exchange
 236             static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
 237             {
 238                 return primary::ret( platform::exchange8( primary::ptr(pDest), primary::val(val), order ));
 239             }
 240             static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) CDS_NOEXCEPT
 241             {
 242                 return primary::ret( platform::exchange8( primary::ptr(pDest), primary::val(val), order ));
 243             }
 244             static T atomic_exchange( T volatile * pDest, T val ) CDS_NOEXCEPT
 245             {
 246                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 247             }
 248             static T atomic_exchange( T * pDest, T val ) CDS_NOEXCEPT
 249             {
 250                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 251             }
 252
 253             // cas
 254             static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 255             {
 256                 assert( expected != NULL );
 257                 return platform::cas8_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 258             }
 259             static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 260             {
 261                 assert( expected != NULL );
 262                 return platform::cas8_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 263             }
 264             static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 265             {
 266                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 267             }
 268             static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 269             {
 270                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 271             }
 272             static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 273             {
 274                 assert( expected != NULL );
 275                 return platform::cas8_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 276             }
 277             static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 278             {
 279                 assert( expected != NULL );
 280                 return platform::cas8_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 281             }
 282             static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 283             {
 284                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 285             }
 286             static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 287             {
 288                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 289             }
 290         };
 291
 292         template <typename T>
 293         struct atomic_integral_ops< T, 1 >
 294             : atomic_generic_ops<T, 1, T >
 295             , atomic_integral_bitwise_ops<T>
 296         {
 297             typedef atomic_integral_bitwise_ops<T> bitwise_ops;
 298
 299             // fetch_add
 300             static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 301             {
 302 #           ifdef CDS_ATOMIC_fetch8_add_defined
 303                 return platform::fetch8_add( pDest, val, order );
 304 #           else
 305                 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
 306                 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
 307                 return cur;
 308 #           endif
 309             }
 310             static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 311             {
 312                 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 313             }
 314             static T atomic_fetch_add( T volatile * pDest, T val ) CDS_NOEXCEPT
 315             {
 316                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 317             }
 318             static T atomic_fetch_add( T * pDest, T val ) CDS_NOEXCEPT
 319             {
 320                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 321             }
 322
 323             // fetch_sub
 324             static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 325             {
 326 #           ifdef CDS_ATOMIC_fetch8_sub_defined
 327                 return platform::fetch8_sub( pDest, val, order );
 328 #           else
 329                 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
 330                 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
 331                 return cur;
 332 #           endif
 333             }
 334             static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 335             {
 336                 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 337             }
 338             static T atomic_fetch_sub( T volatile * pDest, T val ) CDS_NOEXCEPT
 339             {
 340                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
 341             }
 342             static T atomic_fetch_sub( T * pDest, T val ) CDS_NOEXCEPT
 343             {
 344                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
 345             }
 346
 347             // fetch_and
 348             static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 349             {
 350 #           ifdef CDS_ATOMIC_fetch8_and_defined
 351                 return platform::fetch8_and( pDest, val, order );
 352 #           else
 353                 return bitwise_ops::fetch_and( pDest, val, order );
 354 #           endif
 355             }
 356             static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 357             {
 358                 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 359             }
 360             static T atomic_fetch_and( T volatile * pDest, T val ) CDS_NOEXCEPT
 361             {
 362                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
 363             }
 364             static T atomic_fetch_and( T * pDest, T val ) CDS_NOEXCEPT
 365             {
 366                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
 367             }
 368
 369             // fetch_or
 370             static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 371             {
 372 #           ifdef CDS_ATOMIC_fetch8_or_defined
 373                 return platform::fetch8_or( pDest, val, order );
 374 #           else
 375                 return bitwise_ops::fetch_or( pDest, val, order );
 376 #           endif
 377             }
 378             static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 379             {
 380                 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 381             }
 382             static T atomic_fetch_or( T volatile * pDest, T val ) CDS_NOEXCEPT
 383             {
 384                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
 385             }
 386             static T atomic_fetch_or( T * pDest, T val ) CDS_NOEXCEPT
 387             {
 388                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
 389             }
 390
 391             // fetch_xor
 392             static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 393             {
 394 #           ifdef CDS_ATOMIC_fetch8_xor_defined
 395                 return platform::fetch8_xor( pDest, val, order );
 396 #           else
 397                 return bitwise_ops::fetch_xor( pDest, val, order );
 398 #           endif
 399             }
 400             static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 401             {
 402                 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 403             }
 404             static T atomic_fetch_xor( T volatile * pDest, T val ) CDS_NOEXCEPT
 405             {
 406                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
 407             }
 408             static T atomic_fetch_xor( T * pDest, T val ) CDS_NOEXCEPT
 409             {
 410                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
 411             }
 412         };
 413
 414         // 16-bit atomic operations
 415
 416         template <typename T, typename Primary>
 417         struct atomic_generic_ops< T, 2, Primary >
 418         {
 419             typedef make_atomic_primary<T, Primary> primary;
 420
 421             // store
 422             static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
 423             {
 424                 platform::store16( primary::ptr(pDest), primary::val(v), order );
 425             }
 426             static void atomic_store_explicit( T * pDest, T v, memory_order order ) CDS_NOEXCEPT
 427             {
 428                 platform::store16( primary::ptr(pDest), primary::val(v), order );
 429             }
 430             static void atomic_store( T volatile * pDest, T v ) CDS_NOEXCEPT
 431             {
 432                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 433             }
 434             static void atomic_store( T * pDest, T v ) CDS_NOEXCEPT
 435             {
 436                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 437             }
 438
 439             // load
 440             static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
 441             {
 442                 return primary::ret( platform::load16( primary::ptr(pSrc), order ));
 443             }
 444             static T atomic_load_explicit( T const * pSrc, memory_order order ) CDS_NOEXCEPT
 445             {
 446                 return primary::ret( platform::load16( primary::ptr(pSrc), order ));
 447             }
 448             static T atomic_load( T volatile const * pSrc ) CDS_NOEXCEPT
 449             {
 450                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 451             }
 452             static T atomic_load( T const * pSrc ) CDS_NOEXCEPT
 453             {
 454                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 455             }
 456
 457             // exchange
 458             static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
 459             {
 460                 return primary::ret( platform::exchange16( primary::ptr(pDest), primary::val(val), order ));
 461             }
 462             static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) CDS_NOEXCEPT
 463             {
 464                 return primary::ret( platform::exchange16( primary::ptr(pDest), primary::val(val), order ));
 465             }
 466             static T atomic_exchange( T volatile * pDest, T val ) CDS_NOEXCEPT
 467             {
 468                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 469             }
 470             static T atomic_exchange( T * pDest, T val ) CDS_NOEXCEPT
 471             {
 472                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 473             }
 474
 475             // cas
 476             static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 477             {
 478                 assert( expected != NULL );
 479                 return platform::cas16_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 480             }
 481             static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 482             {
 483                 assert( expected != NULL );
 484                 return platform::cas16_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 485             }
 486             static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 487             {
 488                 return atomic_compare_exchange_weak_explicit( pDest, expected, primary::val(desired), memory_order_seq_cst, memory_order_relaxed );
 489             }
 490             static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 491             {
 492                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 493             }
 494             static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 495             {
 496                 assert( expected != NULL );
 497                 return platform::cas16_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 498             }
 499             static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 500             {
 501                 assert( expected != NULL );
 502                 return platform::cas16_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 503             }
 504             static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 505             {
 506                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 507             }
 508             static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 509             {
 510                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 511             }
 512         };
 513
 514         template <typename T>
 515         struct atomic_integral_ops< T, 2 >
 516             : atomic_generic_ops< T, 2, T >
 517             , atomic_integral_bitwise_ops<T>
 518         {
 519             typedef atomic_integral_bitwise_ops<T> bitwise_ops;
 520
 521             // fetch_add
 522             static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 523             {
 524 #           ifdef CDS_ATOMIC_fetch16_add_defined
 525                 return platform::fetch16_add( pDest, val, order );
 526 #           else
 527                 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
 528                 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
 529                 return cur;
 530 #           endif
 531             }
 532             static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 533             {
 534                 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 535             }
 536             static T atomic_fetch_add( T volatile * pDest, T val ) CDS_NOEXCEPT
 537             {
 538                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 539             }
 540             static T atomic_fetch_add( T * pDest, T val ) CDS_NOEXCEPT
 541             {
 542                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 543             }
 544
 545             // fetch_sub
 546             static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 547             {
 548 #           ifdef CDS_ATOMIC_fetch16_sub_defined
 549                 return platform::fetch16_sub( pDest, val, order );
 550 #           else
 551                 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
 552                 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
 553                 return cur;
 554 #           endif
 555             }
 556             static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 557             {
 558                 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 559             }
 560             static T atomic_fetch_sub( T volatile * pDest, T val ) CDS_NOEXCEPT
 561             {
 562                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
 563             }
 564             static T atomic_fetch_sub( T * pDest, T val ) CDS_NOEXCEPT
 565             {
 566                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
 567             }
 568
 569             // fetch_and
 570             static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 571             {
 572 #           ifdef CDS_ATOMIC_fetch16_and_defined
 573                 return platform::fetch16_and( pDest, val, order );
 574 #           else
 575                 return bitwise_ops::fetch_and( pDest, val, order );
 576 #           endif
 577             }
 578             static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 579             {
 580                 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 581             }
 582             static T atomic_fetch_and( T volatile * pDest, T val ) CDS_NOEXCEPT
 583             {
 584                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
 585             }
 586             static T atomic_fetch_and( T * pDest, T val ) CDS_NOEXCEPT
 587             {
 588                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
 589             }
 590
 591             // fetch_or
 592             static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 593             {
 594 #           ifdef CDS_ATOMIC_fetch16_or_defined
 595                 return platform::fetch16_or( pDest, val, order );
 596 #           else
 597                 return bitwise_ops::fetch_or( pDest, val, order );
 598 #           endif
 599             }
 600             static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 601             {
 602                 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 603             }
 604             static T atomic_fetch_or( T volatile * pDest, T val ) CDS_NOEXCEPT
 605             {
 606                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
 607             }
 608             static T atomic_fetch_or( T * pDest, T val ) CDS_NOEXCEPT
 609             {
 610                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
 611             }
 612
 613             // fetch_xor
 614             static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 615             {
 616 #           ifdef CDS_ATOMIC_fetch16_xor_defined
 617                 return platform::fetch16_xor( pDest, val, order );
 618 #           else
 619                 return bitwise_ops::fetch_xor( pDest, val, order );
 620 #           endif
 621             }
 622             static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 623             {
 624                 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 625             }
 626             static T atomic_fetch_xor( T volatile * pDest, T val ) CDS_NOEXCEPT
 627             {
 628                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
 629             }
 630             static T atomic_fetch_xor( T * pDest, T val ) CDS_NOEXCEPT
 631             {
 632                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
 633             }
 634         };
 635
 636         // 32-bit atomic operations
 637
 638         template <typename T, typename Primary>
 639         struct atomic_generic_ops< T, 4, Primary >
 640         {
 641             typedef make_atomic_primary<T, Primary> primary;
 642
 643             // store
 644             static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
 645             {
 646                 platform::store32( primary::ptr(pDest), primary::val(v), order );
 647             }
 648             static void atomic_store_explicit( T * pDest, T v, memory_order order ) CDS_NOEXCEPT
 649             {
 650                 platform::store32( primary::ptr(pDest), primary::val(v), order );
 651             }
 652             static void atomic_store( T volatile * pDest, T v ) CDS_NOEXCEPT
 653             {
 654                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 655             }
 656             static void atomic_store( T * pDest, T v ) CDS_NOEXCEPT
 657             {
 658                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 659             }
 660
 661             // load
 662             static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
 663             {
 664                 return primary::ret( platform::load32( primary::ptr(pSrc), order ));
 665             }
 666             static T atomic_load_explicit( T const * pSrc, memory_order order ) CDS_NOEXCEPT
 667             {
 668                 return primary::ret( platform::load32( primary::ptr(pSrc), order ));
 669             }
 670             static T atomic_load( T volatile const * pSrc ) CDS_NOEXCEPT
 671             {
 672                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 673             }
 674             static T atomic_load( T const * pSrc ) CDS_NOEXCEPT
 675             {
 676                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 677             }
 678
 679             // exchange
 680             static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
 681             {
 682                 return primary::ret( platform::exchange32( primary::ptr(pDest), primary::val(val), order ));
 683             }
 684             static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) CDS_NOEXCEPT
 685             {
 686                 return primary::ret( platform::exchange32( primary::ptr(pDest), primary::val(val), order ));
 687             }
 688             static T atomic_exchange( T volatile * pDest, T val ) CDS_NOEXCEPT
 689             {
 690                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 691             }
 692             static T atomic_exchange( T * pDest, T val ) CDS_NOEXCEPT
 693             {
 694                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 695             }
 696
 697             // cas
 698             static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 699             {
 700                 assert( expected != NULL );
 701                 return platform::cas32_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 702             }
 703             static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 704             {
 705                 assert( expected != NULL );
 706                 return platform::cas32_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 707             }
 708             static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 709             {
 710                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 711             }
 712             static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 713             {
 714                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 715             }
 716             static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 717             {
 718                 assert( expected != NULL );
 719                 return platform::cas32_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 720             }
 721             static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 722             {
 723                 assert( expected != NULL );
 724                 return platform::cas32_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 725             }
 726             static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 727             {
 728                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 729             }
 730             static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 731             {
 732                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 733             }
 734         };
 735
 736         template <typename T>
 737         struct atomic_integral_ops< T, 4 >
 738             : atomic_generic_ops< T, 4, T >
 739             , atomic_integral_bitwise_ops<T>
 740         {
 741             typedef atomic_integral_bitwise_ops<T> bitwise_ops;
 742             // fetch_add
 743             static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 744             {
 745 #           ifdef CDS_ATOMIC_fetch32_add_defined
 746                 return platform::fetch32_add( pDest, val, order );
 747 #           else
 748                 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
 749                 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
 750                 return cur;
 751 #           endif
 752             }
 753             static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 754             {
 755                 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 756             }
 757             static T atomic_fetch_add( T volatile * pDest, T val ) CDS_NOEXCEPT
 758             {
 759                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 760             }
 761             static T atomic_fetch_add( T * pDest, T val ) CDS_NOEXCEPT
 762             {
 763                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 764             }
 765
 766             // fetch_sub
 767             static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 768             {
 769 #           ifdef CDS_ATOMIC_fetch32_sub_defined
 770                 return platform::fetch32_sub( pDest, val, order );
 771 #           else
 772                 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
 773                 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
 774                 return cur;
 775 #           endif
 776             }
 777             static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 778             {
 779                 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 780             }
 781             static T atomic_fetch_sub( T volatile * pDest, T val ) CDS_NOEXCEPT
 782             {
 783                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
 784             }
 785             static T atomic_fetch_sub( T * pDest, T val ) CDS_NOEXCEPT
 786             {
 787                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
 788             }
 789
 790             // fetch_and
 791             static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 792             {
 793 #           ifdef CDS_ATOMIC_fetch32_and_defined
 794                 return platform::fetch32_and( pDest, val, order );
 795 #           else
 796                 return bitwise_ops::fetch_and( pDest, val, order );
 797 #           endif
 798             }
 799             static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 800             {
 801                 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 802             }
 803             static T atomic_fetch_and( T volatile * pDest, T val ) CDS_NOEXCEPT
 804             {
 805                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
 806             }
 807             static T atomic_fetch_and( T * pDest, T val ) CDS_NOEXCEPT
 808             {
 809                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
 810             }
 811
 812             // fetch_or
 813             static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 814             {
 815 #           ifdef CDS_ATOMIC_fetch32_or_defined
 816                 return platform::fetch32_or( pDest, val, order );
 817 #           else
 818                 return bitwise_ops::fetch_or( pDest, val, order );
 819 #           endif
 820             }
 821             static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 822             {
 823                 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 824             }
 825             static T atomic_fetch_or( T volatile * pDest, T val ) CDS_NOEXCEPT
 826             {
 827                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
 828             }
 829             static T atomic_fetch_or( T * pDest, T val ) CDS_NOEXCEPT
 830             {
 831                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
 832             }
 833
 834             // fetch_xor
 835             static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 836             {
 837 #           ifdef CDS_ATOMIC_fetch32_xor_defined
 838                 return platform::fetch32_xor( pDest, val, order );
 839 #           else
 840                 return bitwise_ops::fetch_xor( pDest, val, order );
 841 #           endif
 842             }
 843             static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 844             {
 845                 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 846             }
 847             static T atomic_fetch_xor( T volatile * pDest, T val ) CDS_NOEXCEPT
 848             {
 849                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
 850             }
 851             static T atomic_fetch_xor( T * pDest, T val ) CDS_NOEXCEPT
 852             {
 853                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
 854             }
 855         };
 856
 857
 858         // 64-bit atomic operations
 859
 860         template <typename T, typename Primary>
 861         struct atomic_generic_ops< T, 8, Primary >
 862         {
 863             typedef make_atomic_primary<T, Primary> primary;
 864
 865             // store
 866             static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
 867             {
 868                 platform::store64( primary::ptr(pDest), primary::val(v), order );
 869             }
 870             static void atomic_store_explicit( T * pDest, T v, memory_order order ) CDS_NOEXCEPT
 871             {
 872                 platform::store64( primary::ptr(pDest), primary::val(v), order );
 873             }
 874             static void atomic_store( T volatile * pDest, T v ) CDS_NOEXCEPT
 875             {
 876                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 877             }
 878             static void atomic_store( T * pDest, T v ) CDS_NOEXCEPT
 879             {
 880                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 881             }
 882
 883             // load
 884             static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
 885             {
 886                 return primary::ret( platform::load64( primary::ptr(pSrc), order ));
 887             }
 888             static T atomic_load_explicit( T const * pSrc, memory_order order ) CDS_NOEXCEPT
 889             {
 890                 return primary::ret( platform::load64( primary::ptr(pSrc), order ));
 891             }
 892             static T atomic_load( T volatile const * pSrc ) CDS_NOEXCEPT
 893             {
 894                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 895             }
 896             static T atomic_load( T const * pSrc ) CDS_NOEXCEPT
 897             {
 898                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 899             }
 900
 901             // exchange
 902             static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
 903             {
 904                 return primary::ret( platform::exchange64( primary::ptr(pDest), primary::val(val), order ));
 905             }
 906             static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) CDS_NOEXCEPT
 907             {
 908                 return primary::ret( platform::exchange64( primary::ptr(pDest), primary::val(val), order ));
 909             }
 910             static T atomic_exchange( T volatile * pDest, T val ) CDS_NOEXCEPT
 911             {
 912                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 913             }
 914             static T atomic_exchange( T * pDest, T val ) CDS_NOEXCEPT
 915             {
 916                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 917             }
 918
 919             // cas
 920             static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 921             {
 922                 assert( expected != NULL );
 923                 return platform::cas64_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 924             }
 925             static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 926             {
 927                 assert( expected != NULL );
 928                 return platform::cas64_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 929             }
 930             static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 931             {
 932                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 933             }
 934             static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 935             {
 936                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 937             }
 938             static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 939             {
 940                 assert( expected != NULL );
 941                 return platform::cas64_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 942             }
 943             static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 944             {
 945                 assert( expected != NULL );
 946                 return platform::cas64_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 947             }
 948             static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 949             {
 950                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 951             }
 952             static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 953             {
 954                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 955             }
 956         };
 957
 958
 959         template <typename T>
 960         struct atomic_integral_ops< T, 8 >
 961             : atomic_generic_ops< T, 8, T >
 962             , atomic_integral_bitwise_ops<T>
 963         {
 964             typedef atomic_integral_bitwise_ops<T>  bitwise_ops;
 965             typedef atomic_generic_ops<T, 8, T>     general_ops;
 966
 967             // fetch_add
 968             static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 969             {
 970 #           ifdef CDS_ATOMIC_fetch64_add_defined
 971                 return platform::fetch64_add( pDest, val, order );
 972 #           else
 973                 T cur = general_ops::atomic_load_explicit( pDest, memory_order_relaxed );
 974                 do {} while ( !general_ops::atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
 975                 return cur;
 976 #           endif
 977             }
 978             static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 979             {
 980                 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 981             }
 982             static T atomic_fetch_add( T volatile * pDest, T val ) CDS_NOEXCEPT
 983             {
 984                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 985             }
 986             static T atomic_fetch_add( T * pDest, T val ) CDS_NOEXCEPT
 987             {
 988                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 989             }
 990
 991             // fetch_sub
 992             static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 993             {
 994 #           ifdef CDS_ATOMIC_fetch64_sub_defined
 995                 return platform::fetch64_sub( pDest, val, order );
 996 #           else
 997                 T cur = general_ops::atomic_load_explicit( pDest, memory_order_relaxed );
 998                 do {} while ( !general_ops::atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
 999                 return cur;
1000 #           endif
1001             }
1002             static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
1003             {
1004                 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1005             }
1006             static T atomic_fetch_sub( T volatile * pDest, T val ) CDS_NOEXCEPT
1007             {
1008                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1009             }
1010             static T atomic_fetch_sub( T * pDest, T val ) CDS_NOEXCEPT
1011             {
1012                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1013             }
1014
1015             // fetch_and
1016             static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
1017             {
1018 #           ifdef CDS_ATOMIC_fetch64_and_defined
1019                 return platform::fetch64_and( pDest, val, order );
1020 #           else
1021                 return bitwise_ops::fetch_and( pDest, val, order );
1022 #           endif
1023             }
1024             static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
1025             {
1026                 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1027             }
1028             static T atomic_fetch_and( T volatile * pDest, T val ) CDS_NOEXCEPT
1029             {
1030                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
1031             }
1032             static T atomic_fetch_and( T * pDest, T val ) CDS_NOEXCEPT
1033             {
1034                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
1035             }
1036
1037             // fetch_or
1038             static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
1039             {
1040 #           ifdef CDS_ATOMIC_fetch64_or_defined
1041                 return platform::fetch64_or( pDest, val, order );
1042 #           else
1043                 return bitwise_ops::fetch_or( pDest, val, order );
1044 #           endif
1045             }
1046             static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
1047             {
1048                 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1049             }
1050             static T atomic_fetch_or( T volatile * pDest, T val ) CDS_NOEXCEPT
1051             {
1052                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
1053             }
1054             static T atomic_fetch_or( T * pDest, T val ) CDS_NOEXCEPT
1055             {
1056                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
1057             }
1058
1059             // fetch_xor
1060             static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
1061             {
1062 #           ifdef CDS_ATOMIC_fetch64_xor_defined
1063                 return platform::fetch64_xor( pDest, val, order );
1064 #           else
1065                 return bitwise_ops::fetch_xor( pDest, val, order );
1066 #           endif
1067             }
1068             static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
1069             {
1070                 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1071             }
1072             static T atomic_fetch_xor( T volatile * pDest, T val ) CDS_NOEXCEPT
1073             {
1074                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
1075             }
1076             static T atomic_fetch_xor( T * pDest, T val ) CDS_NOEXCEPT
1077             {
1078                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
1079             }
1080         };
1081
1082
1083         // atomic pointer operations
1084         template <typename T>
1085         struct atomic_pointer_base
1086         {
1087             // store
1088             static void atomic_store_explicit( T * volatile * pDest, T * v, memory_order order ) CDS_NOEXCEPT
1089             {
1090                 platform::store_ptr( pDest, v, order );
1091             }
1092             static void atomic_store_explicit( T * * pDest, T * v, memory_order order ) CDS_NOEXCEPT
1093             {
1094                 platform::store_ptr( pDest, v, order );
1095             }
1096             static void atomic_store( T * volatile * pDest, T * v ) CDS_NOEXCEPT
1097             {
1098                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
1099             }
1100             static void atomic_store( T * * pDest, T * v ) CDS_NOEXCEPT
1101             {
1102                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
1103             }
1104
1105             // load
1106             static T * atomic_load_explicit( T * volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
1107             {
1108                 return platform::load_ptr( pSrc, order );
1109             }
1110             static T * atomic_load_explicit( T * const * pSrc, memory_order order ) CDS_NOEXCEPT
1111             {
1112                 return platform::load_ptr( pSrc, order );
1113             }
1114             static T * atomic_load( T * volatile const * pSrc ) CDS_NOEXCEPT
1115             {
1116                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
1117             }
1118             static T * atomic_load( T * const * pSrc ) CDS_NOEXCEPT
1119             {
1120                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
1121             }
1122
1123             // exchange
1124             static T * atomic_exchange_explicit( T * volatile * pDest, T * val, memory_order order ) CDS_NOEXCEPT
1125             {
1126                 return platform::exchange_ptr( pDest, val, order );
1127             }
1128             static T * atomic_exchange_explicit( T * * pDest, T * val, memory_order order ) CDS_NOEXCEPT
1129             {
1130                 return platform::exchange_ptr( pDest, val, order );
1131             }
1132             static T * atomic_exchange( T * volatile * pDest, T * val ) CDS_NOEXCEPT
1133             {
1134                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
1135             }
1136             static T * atomic_exchange( T * * pDest, T * val ) CDS_NOEXCEPT
1137             {
1138                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
1139             }
1140
1141             // cas
1142             static bool atomic_compare_exchange_weak_explicit( T * volatile * pDest, T * * expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
1143             {
1144                 assert( expected != NULL );
1145                 return platform::cas_ptr_weak( pDest, *expected, desired, mo_success, mo_fail );
1146             }
1147             static bool atomic_compare_exchange_weak_explicit( T * * pDest, T * * expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
1148             {
1149                 assert( expected != NULL );
1150                 return platform::cas_ptr_weak( pDest, *expected, desired, mo_success, mo_fail );
1151             }
1152             static bool atomic_compare_exchange_weak( T * volatile * pDest, T ** expected, T * desired ) CDS_NOEXCEPT
1153             {
1154                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1155             }
1156             static bool atomic_compare_exchange_weak( T ** pDest, T ** expected, T * desired ) CDS_NOEXCEPT
1157             {
1158                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1159             }
1160             static bool atomic_compare_exchange_strong_explicit( T * volatile * pDest, T ** expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
1161             {
1162                 assert( expected != NULL );
1163                 return platform::cas_ptr_strong( pDest, *expected, desired, mo_success, mo_fail );
1164             }
1165             static bool atomic_compare_exchange_strong_explicit( T ** pDest, T ** expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
1166             {
1167                 assert( expected != NULL );
1168                 return platform::cas_ptr_strong( pDest, *expected, desired, mo_success, mo_fail );
1169             }
1170             static bool atomic_compare_exchange_strong( T * volatile * pDest, T ** expected, T * desired ) CDS_NOEXCEPT
1171             {
1172                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1173             }
1174             static bool atomic_compare_exchange_strong( T ** pDest, T ** expected, T * desired ) CDS_NOEXCEPT
1175             {
1176                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1177             }
1178         };
1179
1180         template <typename T>
1181         struct atomic_pointer: public atomic_pointer_base<T>
1182         {
1183             typedef atomic_pointer_base<T> base_class;
1184             // fetch_add
1185             static T * atomic_fetch_add_explicit(T * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
1186             {
1187 #           ifdef CDS_ATOMIC_fetch_ptr_add_defined
1188                 platform::fetch_ptr_add( pDest, val, order );
1189 #           else
1190                 T * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1191                 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
1192                 return cur;
1193 #           endif
1194             }
1195             static T * atomic_fetch_add_explicit(T * * pDest, ptrdiff_t val , memory_order order) CDS_NOEXCEPT
1196             {
1197                 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1198             }
1199             static T * atomic_fetch_add( T * volatile * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1200             {
1201                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1202             }
1203             static T * atomic_fetch_add( T ** pDest, ptrdiff_t val ) CDS_NOEXCEPT
1204             {
1205                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1206             }
1207
1208             // fetch_sub
1209             static T * atomic_fetch_sub_explicit(T * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
1210             {
1211 #           ifdef CDS_ATOMIC_fetch_ptr_sub_defined
1212                 platform::fetch_ptr_sub( pDest, val, order );
1213 #           else
1214                 T * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1215                 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
1216                 return cur;
1217 #           endif
1218             }
1219             static T * atomic_fetch_sub_explicit(T ** pDest, ptrdiff_t val , memory_order order) CDS_NOEXCEPT
1220             {
1221                 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1222             }
1223             static T * atomic_fetch_sub( T volatile * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1224             {
1225                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1226             }
1227             static T * atomic_fetch_sub( T * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1228             {
1229                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1230             }
1231         };
1232
1233         template <>
1234         struct atomic_pointer<void>: public atomic_pointer_base<void>
1235         {
1236             typedef atomic_pointer_base<void>   base_class;
1237
1238             // fetch_add
1239             static void * atomic_fetch_add_explicit(void * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
1240             {
1241                 void * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1242                 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, reinterpret_cast<char *>(cur) + val, order, memory_order_relaxed ));
1243                 return cur;
1244             }
1245             static void * atomic_fetch_add_explicit(void * * pDest, ptrdiff_t val , memory_order order) CDS_NOEXCEPT
1246             {
1247                 return atomic_fetch_add_explicit( reinterpret_cast<void * volatile *>( pDest ), val, order );
1248             }
1249             static void * atomic_fetch_add( void * volatile * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1250             {
1251                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1252             }
1253             static void * atomic_fetch_add( void ** pDest, ptrdiff_t val ) CDS_NOEXCEPT
1254             {
1255                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1256             }
1257
1258             // fetch_sub
1259             static void * atomic_fetch_sub_explicit(void * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
1260             {
1261                 void * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1262                 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, reinterpret_cast<char *>(cur) - val, order, memory_order_relaxed ));
1263                 return cur;
1264             }
1265             static void * atomic_fetch_sub_explicit(void ** pDest, ptrdiff_t val , memory_order order) CDS_NOEXCEPT
1266             {
1267                 return atomic_fetch_sub_explicit( reinterpret_cast<void * volatile *>( pDest ), val, order );
1268             }
1269             static void * atomic_fetch_sub( void * volatile * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1270             {
1271                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1272             }
1273             static void * atomic_fetch_sub( void ** pDest, ptrdiff_t val ) CDS_NOEXCEPT
1274             {
1275                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1276             }
1277         };
1278
1279 #ifndef CDS_CXX11_DELETE_DEFINITION_SUPPORT
1280         class atomic_noncopyable
1281         {
1282         private:
1283             atomic_noncopyable(const atomic_noncopyable&);
1284             atomic_noncopyable& operator=(const atomic_noncopyable&);
1285             //atomic_noncopyable& operator=(const atomic_noncopyable&) volatile;
1286         protected:
1287 #   ifdef CDS_CXX11_EXPLICITLY_DEFAULTED_FUNCTION_SUPPORT
1288             atomic_noncopyable() = default;
1289 #   else
1290             atomic_noncopyable()
1291             {}
1292 #   endif
1293         };
1294 #endif
1295
1296         template <typename T>
1297         struct atomic_integral
1298 #ifndef CDS_CXX11_DELETE_DEFINITION_SUPPORT
1299             : atomic_noncopyable
1300 #endif
1301         {
1302         private:
1303             typename cds::details::aligned_type<T, sizeof(T)>::type volatile m_val;
1304             //T volatile  m_val;
1305             typedef atomic_integral_ops<T, sizeof(T)>   atomic_ops;
1306         public:
1307             typedef T   atomic_type;
1308         public:
1309             bool is_lock_free() const volatile CDS_NOEXCEPT
1310             {
1311                 return true;
1312             }
1313             bool is_lock_free() const CDS_NOEXCEPT
1314             {
1315                 return true;
1316             }
1317             void store(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1318             {
1319                 atomic_ops::atomic_store_explicit( &m_val, val, order );
1320             }
1321             void store(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1322             {
1323                 atomic_ops::atomic_store_explicit( &m_val, val, order );
1324             }
1325
1326             T load(memory_order order = memory_order_seq_cst) const volatile CDS_NOEXCEPT
1327             {
1328                 return atomic_ops::atomic_load_explicit( &m_val, order );
1329             }
1330             T load(memory_order order  = memory_order_seq_cst) const CDS_NOEXCEPT
1331             {
1332                 return atomic_ops::atomic_load_explicit( &m_val, order );
1333             }
1334
1335             operator T() const volatile CDS_NOEXCEPT
1336             {
1337                 return load();
1338             }
1339             operator T() const CDS_NOEXCEPT
1340             {
1341                 return load();
1342             }
1343
1344             T exchange(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1345             {
1346                 return atomic_ops::atomic_exchange_explicit( &m_val, val, order );
1347             }
1348             T exchange(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1349             {
1350                 return atomic_ops::atomic_exchange_explicit( &m_val, val, order );
1351             }
1352
1353             bool compare_exchange_weak(T& expected, T desired , memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1354             {
1355                 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_val, &expected, desired, success_order, failure_order );
1356             }
1357             bool compare_exchange_weak(T& expected, T desired , memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1358             {
1359                 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_val, &expected, desired, success_order, failure_order );
1360             }
1361             bool compare_exchange_strong(T& expected, T desired , memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1362             {
1363                 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_val, &expected, desired, success_order, failure_order );
1364             }
1365             bool compare_exchange_strong(T& expected, T desired , memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1366             {
1367                 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_val, &expected, desired, success_order, failure_order );
1368             }
1369             bool compare_exchange_weak(T& expected, T desired , memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1370             {
1371                 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1372             }
1373             bool compare_exchange_weak(T& expected, T desired , memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1374             {
1375                 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1376             }
1377             bool compare_exchange_strong(T& expected, T desired , memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1378             {
1379                 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1380             }
1381             bool compare_exchange_strong(T& expected, T desired , memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1382             {
1383                 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1384             }
1385
1386             T fetch_add(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1387             {
1388                 return atomic_ops::atomic_fetch_add_explicit( &m_val, val, order );
1389             }
1390             T fetch_add(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1391             {
1392                 return atomic_ops::atomic_fetch_add_explicit( &m_val, val, order );
1393             }
1394             T fetch_sub(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1395             {
1396                 return atomic_ops::atomic_fetch_sub_explicit( &m_val, val, order );
1397             }
1398             T fetch_sub(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1399             {
1400                 return atomic_ops::atomic_fetch_sub_explicit( &m_val, val, order );
1401             }
1402             T fetch_and(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1403             {
1404                 return atomic_ops::atomic_fetch_and_explicit( &m_val, val, order );
1405             }
1406             T fetch_and(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1407             {
1408                 return atomic_ops::atomic_fetch_and_explicit( &m_val, val, order );
1409             }
1410
1411             T fetch_or(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1412             {
1413                 return atomic_ops::atomic_fetch_or_explicit( &m_val, val, order );
1414             }
1415             T fetch_or(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1416             {
1417                 return atomic_ops::atomic_fetch_or_explicit( &m_val, val, order );
1418             }
1419             T fetch_xor(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1420             {
1421                 return atomic_ops::atomic_fetch_xor_explicit( &m_val, val, order );
1422             }
1423             T fetch_xor(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1424             {
1425                 return atomic_ops::atomic_fetch_xor_explicit( &m_val, val, order );
1426             }
1427
1428 #ifdef CDS_CXX11_EXPLICITLY_DEFAULTED_FUNCTION_SUPPORT
1429             atomic_integral() = default;
1430 #else
1431             atomic_integral() CDS_NOEXCEPT
1432             {}
1433 #endif
1434             CDS_CONSTEXPR atomic_integral(T val) CDS_NOEXCEPT
1435                 : m_val(val)
1436                 {}
1437
1438 #ifdef CDS_CXX11_DELETE_DEFINITION_SUPPORT
1439             atomic_integral(const atomic_integral&) = delete;
1440             atomic_integral& operator=(const atomic_integral&) = delete;
1441             atomic_integral& operator=(const atomic_integral&) volatile = delete;
1442 #endif
1443             T operator=(T val) volatile CDS_NOEXCEPT
1444             {
1445                 store(val);
1446                 return val;
1447             }
1448             T operator=(T val) CDS_NOEXCEPT
1449             {
1450                 store(val);
1451                 return val;
1452             }
1453
1454             // Post inc/dec
1455             T operator++(int) volatile CDS_NOEXCEPT
1456             {
1457                 return fetch_add( 1 );
1458             }
1459             T operator++(int) CDS_NOEXCEPT
1460             {
1461                 return fetch_add( 1 );
1462             }
1463             T operator--(int) volatile CDS_NOEXCEPT
1464             {
1465                 return fetch_sub( 1 );
1466             }
1467             T operator--(int) CDS_NOEXCEPT
1468             {
1469                 return fetch_sub( 1 );
1470             }
1471
1472             // Pre inc/dec
1473             T operator++() volatile CDS_NOEXCEPT
1474             {
1475                 return fetch_add( 1 ) + 1;
1476             }
1477             T operator++() CDS_NOEXCEPT
1478             {
1479                 return fetch_add( 1 ) + 1;
1480             }
1481             T operator--() volatile CDS_NOEXCEPT
1482             {
1483                 return fetch_sub( 1 ) - 1;
1484             }
1485             T operator--() CDS_NOEXCEPT
1486             {
1487                 return fetch_sub( 1 ) - 1;
1488             }
1489
1490             // op=
1491             T operator+=(T val) volatile CDS_NOEXCEPT
1492             {
1493                 return fetch_add( val ) + val;
1494             }
1495             T operator+=(T val) CDS_NOEXCEPT
1496             {
1497                 return fetch_add( val ) + val;
1498             }
1499             T operator-=(T val) volatile CDS_NOEXCEPT
1500             {
1501                 return fetch_sub( val ) - val;
1502             }
1503             T operator-=(T val) CDS_NOEXCEPT
1504             {
1505                 return fetch_sub( val ) - val;
1506             }
1507             T operator&=(T val) volatile CDS_NOEXCEPT
1508             {
1509                 return fetch_and( val ) & val;
1510             }
1511             T operator&=(T val) CDS_NOEXCEPT
1512             {
1513                 return fetch_and( val ) & val;
1514             }
1515             T operator|=(T val) volatile CDS_NOEXCEPT
1516             {
1517                 return fetch_or( val ) | val;
1518             }
1519             T operator|=(T val) CDS_NOEXCEPT
1520             {
1521                 return fetch_or( val ) | val;
1522             }
1523             T operator^=(T val) volatile CDS_NOEXCEPT
1524             {
1525                 return fetch_xor( val ) ^ val;
1526             }
1527             T operator^=(T val) CDS_NOEXCEPT
1528             {
1529                 return fetch_xor( val ) ^ val;
1530             }
1531         };
1532
1533         template <typename Type>
1534         struct select_primary_type {
1535             typedef typename details::primary_type<sizeof(Type)>::type type;
1536         };
1537         template <>
1538         struct select_primary_type<bool> {
1539             typedef bool type;
1540         };
1541
1542     }   // namespace details
1543
1544     template <class T>
1545     struct atomic
1546 #ifndef CDS_CXX11_DELETE_DEFINITION_SUPPORT
1547         : details::atomic_noncopyable
1548 #endif
1549     {
1550     private:
1551         typedef details::atomic_generic_ops<T, sizeof(T), typename details::select_primary_type<T>::type >  atomic_ops;
1552
1553         T volatile m_data;
1554     public:
1555         bool is_lock_free() const volatile CDS_NOEXCEPT
1556         {
1557             return true;
1558         }
1559         bool is_lock_free() const CDS_NOEXCEPT
1560         {
1561             return true;
1562         }
1563
1564         void store(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1565         {
1566             atomic_ops::atomic_store_explicit( &m_data, val, order );
1567         }
1568         void store(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1569         {
1570             atomic_ops::atomic_store_explicit( &m_data, val, order );
1571         }
1572
1573         T load(memory_order order = memory_order_seq_cst) const volatile CDS_NOEXCEPT
1574         {
1575             return atomic_ops::atomic_load_explicit( &m_data, order );
1576         }
1577         T load(memory_order order = memory_order_seq_cst) const CDS_NOEXCEPT
1578         {
1579            return atomic_ops::atomic_load_explicit( &m_data, order );
1580         }
1581
1582         operator T() const volatile CDS_NOEXCEPT
1583         {
1584             return load();
1585         }
1586         operator T() const CDS_NOEXCEPT
1587         {
1588             return load();
1589         }
1590
1591         T exchange(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1592         {
1593             return atomic_ops::atomic_exchange_explicit( &m_data, val, order );
1594         }
1595         T exchange(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1596         {
1597             return atomic_ops::atomic_exchange_explicit( &m_data, val, order );
1598         }
1599
1600         bool compare_exchange_weak(T& expected, T desired, memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1601         {
1602             return atomic_ops::atomic_compare_exchange_weak_explicit( &m_data, &expected, desired, success_order, failure_order );
1603         }
1604         bool compare_exchange_weak(T& expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1605         {
1606             return atomic_ops::atomic_compare_exchange_weak_explicit( &m_data, &expected, desired, success_order, failure_order );
1607         }
1608         bool compare_exchange_strong(T& expected, T desired, memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1609         {
1610             return atomic_ops::atomic_compare_exchange_strong_explicit( &m_data, &expected, desired, success_order, failure_order );
1611         }
1612         bool compare_exchange_strong(T& expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1613         {
1614             return atomic_ops::atomic_compare_exchange_strong_explicit( &m_data, &expected, desired, success_order, failure_order );
1615         }
1616         bool compare_exchange_weak(T& expected, T desired, memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1617         {
1618             return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1619         }
1620         bool compare_exchange_weak(T& expected, T desired, memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1621         {
1622             return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1623         }
1624         bool compare_exchange_strong(T& expected, T desired, memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1625         {
1626             return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1627         }
1628         bool compare_exchange_strong(T& expected, T desired, memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1629         {
1630             return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1631         }
1632
1633 #ifdef CDS_CXX11_EXPLICITLY_DEFAULTED_FUNCTION_SUPPORT
1634         atomic() = default;
1635 #else
1636         atomic()
1637         {}
1638 #endif
1639         CDS_CONSTEXPR atomic(T val)
1640             : m_data( val )
1641             {}
1642
1643 #ifdef CDS_CXX11_DELETE_DEFINITION_SUPPORT
1644         atomic(const atomic&) = delete;
1645         atomic& operator=(const atomic&) = delete;
1646         atomic& operator=(const atomic&) volatile = delete;
1647 #endif
1648
1649         T operator=(T val) volatile CDS_NOEXCEPT
1650         {
1651             store( val );
1652             return val;
1653         }
1654         T operator=(T val) CDS_NOEXCEPT
1655         {
1656             store( val );
1657             return val;
1658         }
1659     };
1660
1661 #if defined(CDS_CXX11_EXPLICITLY_DEFAULTED_FUNCTION_SUPPORT) && defined(CDS_CXX11_DELETE_DEFINITION_SUPPORT)
1662 #   define CDS_DECLARE_ATOMIC_INTEGRAL( _type ) \
1663     template <> \
1664     struct atomic<_type>: public details::atomic_integral<_type> \
1665     { \
1666     private: \
1667         typedef details::atomic_integral<_type>   base_class  ; \
1668     public: \
1669         atomic() = default; \
1670         atomic(_type val) CDS_NOEXCEPT : base_class(val) {} \
1671         atomic(const atomic&) = delete; \
1672         atomic& operator=(const atomic&) = delete; \
1673         atomic& operator=(const atomic&) volatile = delete; \
1674         _type operator=(_type val) volatile CDS_NOEXCEPT { return base_class::operator=(val); } \
1675         _type operator=(_type val) CDS_NOEXCEPT { return base_class::operator=(val); } \
1676     };
1677 #else
1678 #   define CDS_DECLARE_ATOMIC_INTEGRAL( _type ) \
1679     template <> \
1680     struct atomic<_type>: public details::atomic_integral<_type> \
1681     { \
1682     private: \
1683         typedef details::atomic_integral<_type>   base_class  ; \
1684     public: \
1685         atomic() {} \
1686         atomic(_type val) CDS_NOEXCEPT : base_class(val) {} \
1687         _type operator=(_type val) volatile CDS_NOEXCEPT { return base_class::operator=(val); } \
1688         _type operator=(_type val) CDS_NOEXCEPT { return base_class::operator=(val); } \
1689     };
1690 #endif
1691
1692     CDS_DECLARE_ATOMIC_INTEGRAL(char)
1693     CDS_DECLARE_ATOMIC_INTEGRAL(signed char)
1694     CDS_DECLARE_ATOMIC_INTEGRAL(unsigned char)
1695     CDS_DECLARE_ATOMIC_INTEGRAL(short)
1696     CDS_DECLARE_ATOMIC_INTEGRAL(unsigned short)
1697     CDS_DECLARE_ATOMIC_INTEGRAL(int)
1698     CDS_DECLARE_ATOMIC_INTEGRAL(unsigned int)
1699     CDS_DECLARE_ATOMIC_INTEGRAL(long)
1700     CDS_DECLARE_ATOMIC_INTEGRAL(unsigned long)
1701     CDS_DECLARE_ATOMIC_INTEGRAL(long long)
1702     CDS_DECLARE_ATOMIC_INTEGRAL(unsigned long long)
1703 //#if CDS_COMPILER == CDS_COMPILER_GCC && CDS_COMPILER_VERSION >= 40400
1704 //    CDS_DECLARE_ATOMIC_INTEGRAL(char16_t)
1705 //    CDS_DECLARE_ATOMIC_INTEGRAL(char32_t)
1706 //#endif
1707 //    CDS_DECLARE_ATOMIC_INTEGRAL(wchar_t)
1708
1709 #   undef CDS_DECLARE_ATOMIC_INTEGRAL
1710
1711
1712     template <typename T>
1713     class atomic<T *>
1714 #ifndef CDS_CXX11_DELETE_DEFINITION_SUPPORT
1715         : details::atomic_noncopyable
1716 #endif
1717     {
1718     private:
1719         T * volatile m_ptr;
1720         typedef details::atomic_pointer<T>  atomic_ops;
1721     public:
1722         bool is_lock_free() const volatile CDS_NOEXCEPT
1723         {
1724             return true;
1725         }
1726         bool is_lock_free() const CDS_NOEXCEPT
1727         {
1728             return true;
1729         }
1730
1731         void store(T * val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1732         {
1733             atomic_ops::atomic_store_explicit( &m_ptr, val, order );
1734         }
1735         void store(T * val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1736         {
1737             atomic_ops::atomic_store_explicit( &m_ptr, val, order );
1738         }
1739
1740         T * load(memory_order order = memory_order_seq_cst) const volatile CDS_NOEXCEPT
1741         {
1742             return atomic_ops::atomic_load_explicit( &m_ptr, order );
1743         }
1744         T * load(memory_order order = memory_order_seq_cst) const CDS_NOEXCEPT
1745         {
1746             return atomic_ops::atomic_load_explicit( &m_ptr, order );
1747         }
1748
1749         operator T *() const volatile CDS_NOEXCEPT
1750         {
1751             return load();
1752         }
1753         operator T *() const CDS_NOEXCEPT
1754         {
1755             return load();
1756         }
1757
1758         T * exchange(T * val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1759         {
1760             return atomic_ops::atomic_exchange_explicit( &m_ptr, val, order );
1761         }
1762         T * exchange(T * val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1763         {
1764             return atomic_ops::atomic_exchange_explicit( &m_ptr, val, order );
1765         }
1766
1767         bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1768         {
1769             return atomic_ops::atomic_compare_exchange_weak_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1770         }
1771         bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1772         {
1773             return atomic_ops::atomic_compare_exchange_weak_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1774         }
1775         bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1776         {
1777             return atomic_ops::atomic_compare_exchange_strong_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1778         }
1779         bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1780         {
1781             return atomic_ops::atomic_compare_exchange_strong_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1782         }
1783         bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1784         {
1785             return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1786         }
1787         bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1788         {
1789             return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1790         }
1791         bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1792         {
1793             return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1794         }
1795         bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1796         {
1797             return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1798         }
1799
1800         T * fetch_add(ptrdiff_t offset, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1801         {
1802             return atomic_ops::atomic_fetch_add_explicit( &m_ptr, offset, order );
1803         }
1804         T * fetch_add(ptrdiff_t offset, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1805         {
1806             return atomic_ops::atomic_fetch_add_explicit( &m_ptr, offset, order );
1807         }
1808
1809         T * fetch_sub(ptrdiff_t offset, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1810         {
1811             return atomic_ops::atomic_fetch_sub_explicit( &m_ptr, offset, order );
1812         }
1813         T * fetch_sub(ptrdiff_t offset, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1814         {
1815             return atomic_ops::atomic_fetch_sub_explicit( &m_ptr, offset, order );
1816         }
1817
1818 #ifdef CDS_CXX11_EXPLICITLY_DEFAULTED_FUNCTION_SUPPORT
1819         atomic() = default;
1820 #else
1821         atomic() CDS_NOEXCEPT
1822         {}
1823 #endif
1824         CDS_CONSTEXPR atomic(T * val) CDS_NOEXCEPT
1825             : m_ptr( val )
1826         {}
1827
1828 #ifdef CDS_CXX11_DELETE_DEFINITION_SUPPORT
1829         atomic(const atomic&) = delete;
1830         atomic& operator=(const atomic&) = delete;
1831         atomic& operator=(const atomic&) volatile = delete;
1832 #endif
1833
1834         T * operator=(T * val) volatile CDS_NOEXCEPT
1835         {
1836             store( val );
1837             return val;
1838         }
1839         T * operator=(T * val) CDS_NOEXCEPT
1840         {
1841             store( val );
1842             return val;
1843         }
1844     };
1845
1846     // Atomic typedefs
1847     typedef atomic<bool>            atomic_bool;
1848     typedef atomic<char>            atomic_char;
1849     typedef atomic<signed char>     atomic_schar;
1850     typedef atomic<unsigned char>   atomic_uchar;
1851     typedef atomic<short>           atomic_short;
1852     typedef atomic<unsigned short>  atomic_ushort;
1853     typedef atomic<int>             atomic_int;
1854     typedef atomic<unsigned int>    atomic_uint;
1855     typedef atomic<long>            atomic_long;
1856     typedef atomic<unsigned long>   atomic_ulong;
1857     typedef atomic<long long>       atomic_llong;
1858     typedef atomic<unsigned long long> atomic_ullong;
1859 #if ( CDS_COMPILER == CDS_COMPILER_GCC && CDS_COMPILER_VERSION >= 40400 ) || CDS_COMPILER == CDS_COMPILER_CLANG
1860     typedef atomic<char16_t>        atomic_char16_t;
1861     typedef atomic<char32_t>        atomic_char32_t;
1862 #endif
1863     typedef atomic<wchar_t>         atomic_wchar_t;
1864
1865
1866     typedef atomic<cds::int_least8_t>    atomic_int_least8_t;
1867     typedef atomic<cds::uint_least8_t>   atomic_uint_least8_t;
1868     typedef atomic<cds::int_least16_t>   atomic_int_least16_t;
1869     typedef atomic<cds::uint_least16_t>  atomic_uint_least16_t;
1870     typedef atomic<cds::int_least32_t>   atomic_int_least32_t;
1871     typedef atomic<cds::uint_least32_t>  atomic_uint_least32_t;
1872     typedef atomic<cds::int_least64_t>   atomic_int_least64_t;
1873     typedef atomic<cds::uint_least64_t>  atomic_uint_least64_t;
1874     typedef atomic<cds::int_fast8_t>     atomic_int_fast8_t;
1875     typedef atomic<cds::uint_fast8_t>    atomic_uint_fast8_t;
1876     typedef atomic<cds::int_fast16_t>    atomic_int_fast16_t;
1877     typedef atomic<cds::uint_fast16_t>   atomic_uint_fast16_t;
1878     typedef atomic<cds::int_fast32_t>    atomic_int_fast32_t;
1879     typedef atomic<cds::uint_fast32_t>   atomic_uint_fast32_t;
1880     typedef atomic<cds::int_fast64_t>    atomic_int_fast64_t;
1881     typedef atomic<cds::uint_fast64_t>   atomic_uint_fast64_t;
1882     typedef atomic<intptr_t>             atomic_intptr_t;
1883     typedef atomic<uintptr_t>            atomic_uintptr_t;
1884     typedef atomic<size_t>               atomic_size_t;
1885     typedef atomic<ptrdiff_t>            atomic_ptrdiff_t;
1886     typedef atomic<cds::intmax_t>        atomic_intmax_t;
1887     typedef atomic<cds::uintmax_t>       atomic_uintmax_t;
1888
1889     template <class T>
1890     static inline bool atomic_is_lock_free(const volatile atomic<T> * p) CDS_NOEXCEPT
1891     {
1892         return p->is_lock_free();
1893     }
1894
1895     template <class T>
1896     static inline bool atomic_is_lock_free(const atomic<T> * p ) CDS_NOEXCEPT
1897     {
1898         return p->is_lock_free();
1899     }
1900
1901     /*
1902     template <class T>
1903     static inline void atomic_init(volatile atomic<T> * p, T val) CDS_NOEXCEPT
1904     {
1905         p->init( val );
1906     }
1907
1908     template <class T>
1909     static inline void atomic_init( atomic<T> * p, T val) CDS_NOEXCEPT
1910     {
1911         p->init( val );
1912     }
1913     */
1914
1915     template <class T>
1916     static inline void atomic_store(volatile atomic<T>* p, T val) CDS_NOEXCEPT
1917     {
1918         p->store(val);
1919     }
1920     template <class T>
1921     static inline void atomic_store(atomic<T>* p, T val) CDS_NOEXCEPT
1922     {
1923         p->store( val );
1924     }
1925
1926     template <class T>
1927     static inline void atomic_store_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1928     {
1929         p->store( val, order );
1930     }
1931     template <class T>
1932     static inline void atomic_store_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1933     {
1934         p->store( val, order );
1935     }
1936
1937     template <class T>
1938     static inline T atomic_load(const volatile atomic<T>* p) CDS_NOEXCEPT
1939     {
1940         return p->load();
1941     }
1942     template <class T>
1943     static inline T atomic_load(const atomic<T>* p) CDS_NOEXCEPT
1944     {
1945         return p->load();
1946     }
1947
1948     template <class T>
1949     static inline T atomic_load_explicit(const volatile atomic<T>* p, memory_order order) CDS_NOEXCEPT
1950     {
1951         return p->load( order );
1952     }
1953     template <class T>
1954     static inline T atomic_load_explicit(const atomic<T>* p, memory_order order) CDS_NOEXCEPT
1955     {
1956         return p->load( order );
1957     }
1958
1959     template <class T>
1960     static inline T atomic_exchange(volatile atomic<T>* p, T val) CDS_NOEXCEPT
1961     {
1962         return p->exchange( val );
1963     }
1964     template <class T>
1965     static inline T atomic_exchange(atomic<T>* p, T val ) CDS_NOEXCEPT
1966     {
1967         return p->exchange( val );
1968     }
1969
1970     template <class T>
1971     static inline T atomic_exchange_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1972     {
1973         return p->exchange( val, order );
1974     }
1975     template <class T>
1976     static inline T atomic_exchange_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1977     {
1978         return p->exchange( val, order );
1979     }
1980
1981     template <class T>
1982     static inline bool atomic_compare_exchange_weak(volatile atomic<T>* p, T* expected, T desired) CDS_NOEXCEPT
1983     {
1984         return p->compare_exchange_weak( *expected, desired );
1985     }
1986     template <class T>
1987     static inline bool atomic_compare_exchange_weak(atomic<T>* p, T* expected, T desired) CDS_NOEXCEPT
1988     {
1989         return p->compare_exchange_weak( *expected, desired );
1990     }
1991
1992     template <class T>
1993     static inline bool atomic_compare_exchange_strong(volatile atomic<T>* p, T* expected, T desired) CDS_NOEXCEPT
1994     {
1995         return p->compare_exchange_strong( *expected, desired );
1996     }
1997     template <class T>
1998     static inline bool atomic_compare_exchange_strong(atomic<T>* p, T* expected, T desired) CDS_NOEXCEPT
1999     {
2000         return p->compare_exchange_strong( *expected, desired );
2001     }
2002
2003     template <class T>
2004     static inline bool atomic_compare_exchange_weak_explicit(volatile atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
2005     {
2006         return p->compare_exchange_weak( *expected, desired, success_order, failure_order );
2007     }
2008     template <class T>
2009     static inline bool atomic_compare_exchange_weak_explicit(atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
2010     {
2011         return p->compare_exchange_weak( *expected, desired, success_order, failure_order );
2012     }
2013
2014     template <class T>
2015     static inline bool atomic_compare_exchange_strong_explicit(volatile atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
2016     {
2017         return p->compare_exchange_strong( *expected, desired, success_order, failure_order );
2018     }
2019     template <class T>
2020     static inline bool atomic_compare_exchange_strong_explicit(atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
2021     {
2022         return p->compare_exchange_strong( *expected, desired, success_order, failure_order );
2023     }
2024
2025     template <class T>
2026     static inline T atomic_fetch_add(volatile atomic<T>* p, T val) CDS_NOEXCEPT
2027     {
2028         return p->fetch_add( val );
2029     }
2030     template <class T>
2031     static inline T atomic_fetch_add(atomic<T>* p, T val) CDS_NOEXCEPT
2032     {
2033         return p->fetch_add( val );
2034     }
2035     template <class T>
2036     static inline T * atomic_fetch_add(volatile atomic<T *>* p, ptrdiff_t offset) CDS_NOEXCEPT
2037     {
2038         return p->fetch_add( offset );
2039     }
2040     template <class T>
2041     static inline T * atomic_fetch_add(atomic<T *>* p, ptrdiff_t offset) CDS_NOEXCEPT
2042     {
2043         return p->fetch_add( offset );
2044     }
2045
2046     template <class T>
2047     static inline T atomic_fetch_add_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2048     {
2049         return p->fetch_add( val, order );
2050     }
2051     template <class T>
2052     static inline T atomic_fetch_add_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2053     {
2054         return p->fetch_add( val, order );
2055     }
2056     template <class T>
2057     static inline T * atomic_fetch_add_explicit(volatile atomic<T *>* p, ptrdiff_t offset, memory_order order) CDS_NOEXCEPT
2058     {
2059         return p->fetch_add( offset, order );
2060     }
2061     template <class T>
2062     static inline T * atomic_fetch_add_explicit(atomic<T *>* p, ptrdiff_t offset, memory_order order) CDS_NOEXCEPT
2063     {
2064         return p->fetch_add( offset, order );
2065     }
2066
2067     template <class T>
2068     static inline T atomic_fetch_sub(volatile atomic<T>* p, T val) CDS_NOEXCEPT
2069     {
2070         return p->fetch_sub( val );
2071     }
2072     template <class T>
2073     static inline T atomic_fetch_sub(atomic<T>* p, T val) CDS_NOEXCEPT
2074     {
2075         return p->fetch_sub( val );
2076     }
2077     template <class T>
2078     static inline T * atomic_fetch_sub(volatile atomic<T *>* p, ptrdiff_t offset) CDS_NOEXCEPT
2079     {
2080         return p->fetch_sub( offset );
2081     }
2082     template <class T>
2083     static inline T * atomic_fetch_sub(atomic<T *>* p, ptrdiff_t offset) CDS_NOEXCEPT
2084     {
2085         return p->fetch_sub( offset );
2086     }
2087
2088     template <class T>
2089     static inline T atomic_fetch_sub_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2090     {
2091         return p->fetch_sub( val, order );
2092     }
2093     template <class T>
2094     static inline T atomic_fetch_sub_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2095     {
2096         return p->fetch_sub( val, order );
2097     }
2098     template <class T>
2099     static inline T * atomic_fetch_sub_explicit(volatile atomic<T *>* p, ptrdiff_t offset, memory_order order) CDS_NOEXCEPT
2100     {
2101         return p->fetch_sub( offset, order );
2102     }
2103     template <class T>
2104     static inline T * atomic_fetch_sub_explicit(atomic<T *>* p, ptrdiff_t offset, memory_order order) CDS_NOEXCEPT
2105     {
2106         return p->fetch_sub( offset, order );
2107     }
2108
2109     template <class T>
2110     static inline T atomic_fetch_and(volatile atomic<T>* p, T val) CDS_NOEXCEPT
2111     {
2112         return p->fetch_and( val );
2113     }
2114     template <class T>
2115     static inline T atomic_fetch_and(atomic<T>* p, T val) CDS_NOEXCEPT
2116     {
2117         return p->fetch_and( val );
2118     }
2119
2120     template <class T>
2121     static inline T atomic_fetch_and_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2122     {
2123         return p->fetch_and( val, order );
2124     }
2125     template <class T>
2126     static inline T atomic_fetch_and_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2127     {
2128         return p->fetch_and( val, order );
2129     }
2130
2131     template <class T>
2132     static inline T atomic_fetch_or(volatile atomic<T>* p, T val) CDS_NOEXCEPT
2133     {
2134         return p->fetch_or( val );
2135     }
2136     template <class T>
2137     static inline T atomic_fetch_or(atomic<T>* p, T val) CDS_NOEXCEPT
2138     {
2139         return p->fetch_or( val );
2140     }
2141
2142     template <class T>
2143     static inline T atomic_fetch_or_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2144     {
2145         return p->fetch_or( val, order );
2146     }
2147     template <class T>
2148     static inline T atomic_fetch_or_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2149     {
2150         return p->fetch_or( val, order );
2151     }
2152
2153     template <class T>
2154     static inline T atomic_fetch_xor(volatile atomic<T>* p, T val) CDS_NOEXCEPT
2155     {
2156         return p->fetch_xor( val );
2157     }
2158     template <class T>
2159     static inline T atomic_fetch_xor(atomic<T>* p, T val) CDS_NOEXCEPT
2160     {
2161         return p->fetch_xor( val );
2162     }
2163
2164     template <class T>
2165     static inline T atomic_fetch_xor_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2166     {
2167         return p->fetch_xor( val, order );
2168     }
2169     template <class T>
2170     static inline T atomic_fetch_xor_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2171     {
2172         return p->fetch_xor( val, order );
2173     }
2174
2175     // Atomic flag type
2176     typedef struct atomic_flag
2177     {
2178         void clear( memory_order order = memory_order_seq_cst ) volatile CDS_NOEXCEPT
2179         {
2180             assert( order != memory_order_acquire
2181                 && order != memory_order_acq_rel
2182                 && order != memory_order_consume
2183                 );
2184             platform::atomic_flag_clear( &m_Flag, order );
2185         }
2186         void clear( memory_order order = memory_order_seq_cst ) CDS_NOEXCEPT
2187         {
2188             assert( order != memory_order_acquire
2189                 && order != memory_order_acq_rel
2190                 && order != memory_order_consume
2191                 );
2192             platform::atomic_flag_clear( &m_Flag, order );
2193         }
2194
2195         bool test_and_set( memory_order order = memory_order_seq_cst ) volatile CDS_NOEXCEPT
2196         {
2197             return platform::atomic_flag_tas( &m_Flag, order );
2198         }
2199         bool test_and_set( memory_order order = memory_order_seq_cst ) CDS_NOEXCEPT
2200         {
2201             return platform::atomic_flag_tas( &m_Flag, order );
2202         }
2203
2204 #ifdef CDS_CXX11_EXPLICITLY_DEFAULTED_FUNCTION_SUPPORT
2205         atomic_flag() = default;
2206 #elif CDS_COMPILER != CDS_COMPILER_MSVC
2207         // MS VC generate error C2552 "non-aggregates cannot be initialized with initializer list"
2208         // when atomic_flag initializes with ATOMIC_FLAG_INIT
2209         atomic_flag()
2210         {}
2211 #endif
2212
2213 #ifdef CDS_CXX11_DELETE_DEFINITION_SUPPORT
2214         atomic_flag(const atomic_flag&) = delete;
2215         atomic_flag& operator=(const atomic_flag&) = delete;
2216         atomic_flag& operator=(const atomic_flag&) volatile = delete;
2217 #elif CDS_COMPILER != CDS_COMPILER_MSVC
2218     // MS VC generate error C2552 "non-aggregates cannot be initialized with initializer list"
2219     // when atomic_flag initializes with ATOMIC_FLAG_INIT
2220     private:
2221         atomic_flag(const atomic_flag&);
2222         atomic_flag& operator=(const atomic_flag&);
2223         atomic_flag& operator=(const atomic_flag&) volatile;
2224     public:
2225 #endif
2226
2227         platform::atomic_flag_type volatile m_Flag;
2228     } atomic_flag;
2229
2230     static inline bool atomic_flag_test_and_set(volatile atomic_flag* p) CDS_NOEXCEPT
2231     {
2232         return p->test_and_set();
2233     }
2234     static inline bool atomic_flag_test_and_set(atomic_flag * p) CDS_NOEXCEPT
2235     {
2236         return p->test_and_set();
2237     }
2238     static inline bool atomic_flag_test_and_set_explicit(volatile atomic_flag* p, memory_order order) CDS_NOEXCEPT
2239     {
2240         return p->test_and_set( order );
2241     }
2242     static inline bool atomic_flag_test_and_set_explicit(atomic_flag* p, memory_order order) CDS_NOEXCEPT
2243     {
2244         return p->test_and_set( order );
2245     }
2246     static inline void atomic_flag_clear(volatile atomic_flag* p) CDS_NOEXCEPT
2247     {
2248         return p->clear();
2249     }
2250     static inline void atomic_flag_clear(atomic_flag* p) CDS_NOEXCEPT
2251     {
2252         return p->clear();
2253     }
2254     static inline void atomic_flag_clear_explicit(volatile atomic_flag* p, memory_order order) CDS_NOEXCEPT
2255     {
2256         return p->clear( order );
2257     }
2258     static inline void atomic_flag_clear_explicit(atomic_flag* p, memory_order order) CDS_NOEXCEPT
2259     {
2260         return p->clear( order );
2261     }
2262
2263     // Fences
2264     static inline void atomic_thread_fence(memory_order order) CDS_NOEXCEPT
2265     {
2266         platform::thread_fence( order );
2267         CDS_COMPILER_RW_BARRIER;
2268     }
2269     static inline void atomic_signal_fence(memory_order order) CDS_NOEXCEPT
2270     {
2271         platform::signal_fence( order );
2272     }
2273
2274 }}  // namespace cds::cxx11_atomics
2275
2276 //@endcond
2277 #endif // #ifndef __CDS_COMPILER_CXX11_ATOMIC_H