cds/compiler/cxx11_atomic.h

   1 //$$CDS-header$$
   2
   3 #ifndef __CDS_COMPILER_CXX11_ATOMIC_H
   4 #define __CDS_COMPILER_CXX11_ATOMIC_H
   5 //@cond
   6
   7 #include <cds/details/defs.h>
   8 #include <cds/details/aligned_type.h>
   9
  10 namespace cds { namespace cxx11_atomic {
  11     typedef enum memory_order {
  12         memory_order_relaxed,
  13         memory_order_consume,
  14         memory_order_acquire,
  15         memory_order_release,
  16         memory_order_acq_rel,
  17         memory_order_seq_cst
  18     } memory_order;
  19
  20 }}  // namespace cds::cxx11_atomic
  21
  22
  23 #if CDS_COMPILER == CDS_COMPILER_MSVC || (CDS_COMPILER == CDS_COMPILER_INTEL && CDS_OS_INTERFACE == CDS_OSI_WINDOWS)
  24 #   if CDS_PROCESSOR_ARCH == CDS_PROCESSOR_X86
  25 #       include <cds/compiler/vc/x86/cxx11_atomic.h>
  26 #   elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_AMD64
  27 #       include <cds/compiler/vc/amd64/cxx11_atomic.h>
  28 #   else
  29 #       error "MS VC++ compiler: unsupported processor architecture"
  30 #   endif
  31 #elif CDS_COMPILER == CDS_COMPILER_GCC || CDS_COMPILER == CDS_COMPILER_CLANG || CDS_COMPILER == CDS_COMPILER_INTEL
  32 #   if CDS_PROCESSOR_ARCH == CDS_PROCESSOR_X86
  33 #       include <cds/compiler/gcc/x86/cxx11_atomic.h>
  34 #   elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_AMD64
  35 #       include <cds/compiler/gcc/amd64/cxx11_atomic.h>
  36 #   elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_IA64
  37 #       include <cds/compiler/gcc/ia64/cxx11_atomic.h>
  38 #   elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_SPARC
  39 #       include <cds/compiler/gcc/sparc/cxx11_atomic.h>
  40 #   elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_PPC64
  41 #       include <cds/compiler/gcc/ppc64/cxx11_atomic.h>
  42 //#   elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_ARM7
  43 //#       include <cds/compiler/gcc/arm7/cxx11_atomic.h>
  44 #   else
  45 #       error "GCC compiler: unsupported processor architecture. Try to use native C++11 atomic or boost.atomic"
  46 #   endif
  47 #else
  48 #   error "Undefined compiler"
  49 #endif
  50
  51 // In C++11, make_unsigned is declared in <type_traits>
  52 #include <boost/type_traits/make_unsigned.hpp>  // for make_unsigned
  53
  54 namespace cds { namespace cxx11_atomic {
  55
  56     // forward declarations
  57     template <class T>
  58     struct atomic;
  59
  60     namespace details {
  61
  62         template <typename T, size_t Size, typename Primary = T >
  63         struct atomic_generic_ops;
  64
  65         template <typename T, size_t Size>
  66         struct atomic_integral_ops;
  67
  68         template <size_t TypeSize>
  69         struct primary_type;
  70
  71         template <>
  72         struct primary_type<1>
  73         {
  74             typedef cds::uint8_t type;
  75         };
  76         template <>
  77         struct primary_type<2>
  78         {
  79             typedef cds::uint16_t type;
  80         };
  81         template <>
  82         struct primary_type<4>
  83         {
  84             typedef cds::uint32_t type;
  85         };
  86         template <>
  87         struct primary_type<8>
  88         {
  89             typedef cds::uint64_t type;
  90         };
  91
  92         template <typename T, typename Primary>
  93         struct make_atomic_primary
  94         {
  95             typedef T       source_type;
  96             typedef Primary primary_type;
  97
  98             static primary_type volatile * ptr( source_type volatile * p ) CDS_NOEXCEPT
  99             {
 100                 return reinterpret_cast<primary_type volatile *>(p);
 101             }
 102             static primary_type const volatile * ptr( source_type const volatile * p ) CDS_NOEXCEPT
 103             {
 104                 return reinterpret_cast<primary_type const volatile *>(p);
 105             }
 106
 107             static primary_type val( source_type v ) CDS_NOEXCEPT
 108             {
 109                 return *reinterpret_cast<primary_type*>(&v);
 110             }
 111
 112             static primary_type& ref( source_type& v ) CDS_NOEXCEPT
 113             {
 114                 return reinterpret_cast<primary_type&>(v);
 115             }
 116
 117             static primary_type const& ref( source_type const& v ) CDS_NOEXCEPT
 118             {
 119                 return reinterpret_cast<primary_type const&>(v);
 120             }
 121
 122             static source_type ret( primary_type r ) CDS_NOEXCEPT
 123             {
 124                 return *reinterpret_cast<source_type *>(&r);
 125             }
 126         };
 127
 128         template <typename T>
 129         struct make_atomic_primary<T, T>
 130         {
 131             typedef T source_type;
 132             typedef T primary_type;
 133
 134             static primary_type volatile * ptr( source_type volatile * p ) CDS_NOEXCEPT
 135             {
 136                 return p;
 137             }
 138             static primary_type const volatile * ptr( source_type const volatile * p ) CDS_NOEXCEPT
 139             {
 140                 return p;
 141             }
 142
 143             static primary_type val( source_type v ) CDS_NOEXCEPT
 144             {
 145                 return v;
 146             }
 147
 148             static primary_type& ref( source_type& v ) CDS_NOEXCEPT
 149             {
 150                 return v;
 151             }
 152
 153             static source_type ret( primary_type r ) CDS_NOEXCEPT
 154             {
 155                 return r;
 156             }
 157         };
 158
 159         template <typename T>
 160         struct atomic_integral_bitwise_ops
 161         {
 162         public:
 163             typedef typename boost::make_unsigned<T>::type unsigned_type;
 164             typedef atomic_generic_ops<unsigned_type, sizeof(unsigned_type)> atomic_ops;
 165
 166             static T fetch_and(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 167             {
 168                 unsigned_type cur = atomic_ops::atomic_load_explicit( reinterpret_cast<unsigned_type volatile *>(pDest), memory_order_relaxed );
 169                 do {} while ( !atomic_ops::atomic_compare_exchange_weak_explicit(
 170                     reinterpret_cast<unsigned_type volatile *>(pDest), &cur, cur & unsigned_type(val), order, memory_order_relaxed ));
 171                 return T(cur);
 172             }
 173
 174             static T fetch_or(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 175             {
 176                 unsigned_type cur = atomic_ops::atomic_load_explicit( reinterpret_cast<unsigned_type volatile *>(pDest), memory_order_relaxed );
 177                 do {} while ( !atomic_ops::atomic_compare_exchange_weak_explicit(
 178                     reinterpret_cast<unsigned_type volatile *>(pDest), &cur, cur | unsigned_type(val), order, memory_order_relaxed ));
 179                 return T(cur);
 180             }
 181
 182             static T fetch_xor(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 183             {
 184                 unsigned_type cur = atomic_ops::atomic_load_explicit( reinterpret_cast<unsigned_type volatile *>(pDest), memory_order_relaxed );
 185                 do {} while ( !atomic_ops::atomic_compare_exchange_weak_explicit(
 186                     reinterpret_cast<unsigned_type volatile *>(pDest), &cur, cur ^ unsigned_type(val), order, memory_order_relaxed ));
 187                 return T(cur);
 188             }
 189         };
 190
 191
 192         // 8-bit atomic operations
 193
 194         template <typename T, typename Primary>
 195         struct atomic_generic_ops< T, 1, Primary >
 196         {
 197             typedef make_atomic_primary<T, Primary> primary;
 198
 199             // store
 200             static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
 201             {
 202                 platform::store8( primary::ptr(pDest), primary::val(v), order );
 203             }
 204             static void atomic_store_explicit( T * pDest, T v, memory_order order ) CDS_NOEXCEPT
 205             {
 206                 platform::store8( primary::ptr(pDest), primary::val(v), order );
 207             }
 208             static void atomic_store( T volatile * pDest, T v ) CDS_NOEXCEPT
 209             {
 210                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 211             }
 212             static void atomic_store( T * pDest, T v ) CDS_NOEXCEPT
 213             {
 214                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 215             }
 216
 217             // load
 218             static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
 219             {
 220                 return primary::ret( platform::load8( primary::ptr(pSrc), order ));
 221             }
 222             static T atomic_load_explicit( T const * pSrc, memory_order order ) CDS_NOEXCEPT
 223             {
 224                 return primary::ret( platform::load8( primary::ptr(pSrc), order ));
 225             }
 226             static T atomic_load( T volatile const * pSrc ) CDS_NOEXCEPT
 227             {
 228                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 229             }
 230             static T atomic_load( T const * pSrc ) CDS_NOEXCEPT
 231             {
 232                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 233             }
 234
 235             // exchange
 236             static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
 237             {
 238                 return primary::ret( platform::exchange8( primary::ptr(pDest), primary::val(val), order ));
 239             }
 240             static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) CDS_NOEXCEPT
 241             {
 242                 return primary::ret( platform::exchange8( primary::ptr(pDest), primary::val(val), order ));
 243             }
 244             static T atomic_exchange( T volatile * pDest, T val ) CDS_NOEXCEPT
 245             {
 246                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 247             }
 248             static T atomic_exchange( T * pDest, T val ) CDS_NOEXCEPT
 249             {
 250                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 251             }
 252
 253             // cas
 254             static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 255             {
 256                 assert( expected );
 257                 return platform::cas8_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 258             }
 259             static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 260             {
 261                 assert( expected );
 262                 return platform::cas8_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 263             }
 264             static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 265             {
 266                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 267             }
 268             static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 269             {
 270                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 271             }
 272             static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 273             {
 274                 assert( expected );
 275                 return platform::cas8_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 276             }
 277             static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 278             {
 279                 assert( expected );
 280                 return platform::cas8_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 281             }
 282             static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 283             {
 284                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 285             }
 286             static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 287             {
 288                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 289             }
 290         };
 291
 292         template <typename T>
 293         struct atomic_integral_ops< T, 1 >
 294             : atomic_generic_ops<T, 1, T >
 295             , atomic_integral_bitwise_ops<T>
 296         {
 297             typedef atomic_integral_bitwise_ops<T> bitwise_ops;
 298
 299             // fetch_add
 300             static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 301             {
 302 #           ifdef CDS_ATOMIC_fetch8_add_defined
 303                 return platform::fetch8_add( pDest, val, order );
 304 #           else
 305                 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
 306                 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
 307                 return cur;
 308 #           endif
 309             }
 310             static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 311             {
 312                 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 313             }
 314             static T atomic_fetch_add( T volatile * pDest, T val ) CDS_NOEXCEPT
 315             {
 316                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 317             }
 318             static T atomic_fetch_add( T * pDest, T val ) CDS_NOEXCEPT
 319             {
 320                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 321             }
 322
 323             // fetch_sub
 324             static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 325             {
 326 #           ifdef CDS_ATOMIC_fetch8_sub_defined
 327                 return platform::fetch8_sub( pDest, val, order );
 328 #           else
 329                 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
 330                 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
 331                 return cur;
 332 #           endif
 333             }
 334             static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 335             {
 336                 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 337             }
 338             static T atomic_fetch_sub( T volatile * pDest, T val ) CDS_NOEXCEPT
 339             {
 340                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
 341             }
 342             static T atomic_fetch_sub( T * pDest, T val ) CDS_NOEXCEPT
 343             {
 344                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
 345             }
 346
 347             // fetch_and
 348             static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 349             {
 350 #           ifdef CDS_ATOMIC_fetch8_and_defined
 351                 return platform::fetch8_and( pDest, val, order );
 352 #           else
 353                 return bitwise_ops::fetch_and( pDest, val, order );
 354 #           endif
 355             }
 356             static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 357             {
 358                 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 359             }
 360             static T atomic_fetch_and( T volatile * pDest, T val ) CDS_NOEXCEPT
 361             {
 362                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
 363             }
 364             static T atomic_fetch_and( T * pDest, T val ) CDS_NOEXCEPT
 365             {
 366                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
 367             }
 368
 369             // fetch_or
 370             static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 371             {
 372 #           ifdef CDS_ATOMIC_fetch8_or_defined
 373                 return platform::fetch8_or( pDest, val, order );
 374 #           else
 375                 return bitwise_ops::fetch_or( pDest, val, order );
 376 #           endif
 377             }
 378             static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 379             {
 380                 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 381             }
 382             static T atomic_fetch_or( T volatile * pDest, T val ) CDS_NOEXCEPT
 383             {
 384                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
 385             }
 386             static T atomic_fetch_or( T * pDest, T val ) CDS_NOEXCEPT
 387             {
 388                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
 389             }
 390
 391             // fetch_xor
 392             static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 393             {
 394 #           ifdef CDS_ATOMIC_fetch8_xor_defined
 395                 return platform::fetch8_xor( pDest, val, order );
 396 #           else
 397                 return bitwise_ops::fetch_xor( pDest, val, order );
 398 #           endif
 399             }
 400             static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 401             {
 402                 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 403             }
 404             static T atomic_fetch_xor( T volatile * pDest, T val ) CDS_NOEXCEPT
 405             {
 406                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
 407             }
 408             static T atomic_fetch_xor( T * pDest, T val ) CDS_NOEXCEPT
 409             {
 410                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
 411             }
 412         };
 413
 414         // 16-bit atomic operations
 415
 416         template <typename T, typename Primary>
 417         struct atomic_generic_ops< T, 2, Primary >
 418         {
 419             typedef make_atomic_primary<T, Primary> primary;
 420
 421             // store
 422             static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
 423             {
 424                 platform::store16( primary::ptr(pDest), primary::val(v), order );
 425             }
 426             static void atomic_store_explicit( T * pDest, T v, memory_order order ) CDS_NOEXCEPT
 427             {
 428                 platform::store16( primary::ptr(pDest), primary::val(v), order );
 429             }
 430             static void atomic_store( T volatile * pDest, T v ) CDS_NOEXCEPT
 431             {
 432                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 433             }
 434             static void atomic_store( T * pDest, T v ) CDS_NOEXCEPT
 435             {
 436                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 437             }
 438
 439             // load
 440             static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
 441             {
 442                 return primary::ret( platform::load16( primary::ptr(pSrc), order ));
 443             }
 444             static T atomic_load_explicit( T const * pSrc, memory_order order ) CDS_NOEXCEPT
 445             {
 446                 return primary::ret( platform::load16( primary::ptr(pSrc), order ));
 447             }
 448             static T atomic_load( T volatile const * pSrc ) CDS_NOEXCEPT
 449             {
 450                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 451             }
 452             static T atomic_load( T const * pSrc ) CDS_NOEXCEPT
 453             {
 454                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 455             }
 456
 457             // exchange
 458             static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
 459             {
 460                 return primary::ret( platform::exchange16( primary::ptr(pDest), primary::val(val), order ));
 461             }
 462             static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) CDS_NOEXCEPT
 463             {
 464                 return primary::ret( platform::exchange16( primary::ptr(pDest), primary::val(val), order ));
 465             }
 466             static T atomic_exchange( T volatile * pDest, T val ) CDS_NOEXCEPT
 467             {
 468                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 469             }
 470             static T atomic_exchange( T * pDest, T val ) CDS_NOEXCEPT
 471             {
 472                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 473             }
 474
 475             // cas
 476             static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 477             {
 478                 assert( expected );
 479                 return platform::cas16_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 480             }
 481             static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 482             {
 483                 assert( expected );
 484                 return platform::cas16_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 485             }
 486             static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 487             {
 488                 return atomic_compare_exchange_weak_explicit( pDest, expected, primary::val(desired), memory_order_seq_cst, memory_order_relaxed );
 489             }
 490             static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 491             {
 492                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 493             }
 494             static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 495             {
 496                 assert( expected );
 497                 return platform::cas16_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 498             }
 499             static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 500             {
 501                 assert( expected );
 502                 return platform::cas16_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 503             }
 504             static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 505             {
 506                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 507             }
 508             static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 509             {
 510                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 511             }
 512         };
 513
 514         template <typename T>
 515         struct atomic_integral_ops< T, 2 >
 516             : atomic_generic_ops< T, 2, T >
 517             , atomic_integral_bitwise_ops<T>
 518         {
 519             typedef atomic_integral_bitwise_ops<T> bitwise_ops;
 520
 521             // fetch_add
 522             static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 523             {
 524 #           ifdef CDS_ATOMIC_fetch16_add_defined
 525                 return platform::fetch16_add( pDest, val, order );
 526 #           else
 527                 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
 528                 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
 529                 return cur;
 530 #           endif
 531             }
 532             static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 533             {
 534                 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 535             }
 536             static T atomic_fetch_add( T volatile * pDest, T val ) CDS_NOEXCEPT
 537             {
 538                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 539             }
 540             static T atomic_fetch_add( T * pDest, T val ) CDS_NOEXCEPT
 541             {
 542                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 543             }
 544
 545             // fetch_sub
 546             static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 547             {
 548 #           ifdef CDS_ATOMIC_fetch16_sub_defined
 549                 return platform::fetch16_sub( pDest, val, order );
 550 #           else
 551                 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
 552                 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
 553                 return cur;
 554 #           endif
 555             }
 556             static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 557             {
 558                 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 559             }
 560             static T atomic_fetch_sub( T volatile * pDest, T val ) CDS_NOEXCEPT
 561             {
 562                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
 563             }
 564             static T atomic_fetch_sub( T * pDest, T val ) CDS_NOEXCEPT
 565             {
 566                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
 567             }
 568
 569             // fetch_and
 570             static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 571             {
 572 #           ifdef CDS_ATOMIC_fetch16_and_defined
 573                 return platform::fetch16_and( pDest, val, order );
 574 #           else
 575                 return bitwise_ops::fetch_and( pDest, val, order );
 576 #           endif
 577             }
 578             static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 579             {
 580                 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 581             }
 582             static T atomic_fetch_and( T volatile * pDest, T val ) CDS_NOEXCEPT
 583             {
 584                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
 585             }
 586             static T atomic_fetch_and( T * pDest, T val ) CDS_NOEXCEPT
 587             {
 588                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
 589             }
 590
 591             // fetch_or
 592             static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 593             {
 594 #           ifdef CDS_ATOMIC_fetch16_or_defined
 595                 return platform::fetch16_or( pDest, val, order );
 596 #           else
 597                 return bitwise_ops::fetch_or( pDest, val, order );
 598 #           endif
 599             }
 600             static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 601             {
 602                 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 603             }
 604             static T atomic_fetch_or( T volatile * pDest, T val ) CDS_NOEXCEPT
 605             {
 606                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
 607             }
 608             static T atomic_fetch_or( T * pDest, T val ) CDS_NOEXCEPT
 609             {
 610                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
 611             }
 612
 613             // fetch_xor
 614             static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 615             {
 616 #           ifdef CDS_ATOMIC_fetch16_xor_defined
 617                 return platform::fetch16_xor( pDest, val, order );
 618 #           else
 619                 return bitwise_ops::fetch_xor( pDest, val, order );
 620 #           endif
 621             }
 622             static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 623             {
 624                 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 625             }
 626             static T atomic_fetch_xor( T volatile * pDest, T val ) CDS_NOEXCEPT
 627             {
 628                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
 629             }
 630             static T atomic_fetch_xor( T * pDest, T val ) CDS_NOEXCEPT
 631             {
 632                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
 633             }
 634         };
 635
 636         // 32-bit atomic operations
 637
 638         template <typename T, typename Primary>
 639         struct atomic_generic_ops< T, 4, Primary >
 640         {
 641             typedef make_atomic_primary<T, Primary> primary;
 642
 643             // store
 644             static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
 645             {
 646                 platform::store32( primary::ptr(pDest), primary::val(v), order );
 647             }
 648             static void atomic_store_explicit( T * pDest, T v, memory_order order ) CDS_NOEXCEPT
 649             {
 650                 platform::store32( primary::ptr(pDest), primary::val(v), order );
 651             }
 652             static void atomic_store( T volatile * pDest, T v ) CDS_NOEXCEPT
 653             {
 654                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 655             }
 656             static void atomic_store( T * pDest, T v ) CDS_NOEXCEPT
 657             {
 658                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 659             }
 660
 661             // load
 662             static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
 663             {
 664                 return primary::ret( platform::load32( primary::ptr(pSrc), order ));
 665             }
 666             static T atomic_load_explicit( T const * pSrc, memory_order order ) CDS_NOEXCEPT
 667             {
 668                 return primary::ret( platform::load32( primary::ptr(pSrc), order ));
 669             }
 670             static T atomic_load( T volatile const * pSrc ) CDS_NOEXCEPT
 671             {
 672                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 673             }
 674             static T atomic_load( T const * pSrc ) CDS_NOEXCEPT
 675             {
 676                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 677             }
 678
 679             // exchange
 680             static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
 681             {
 682                 return primary::ret( platform::exchange32( primary::ptr(pDest), primary::val(val), order ));
 683             }
 684             static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) CDS_NOEXCEPT
 685             {
 686                 return primary::ret( platform::exchange32( primary::ptr(pDest), primary::val(val), order ));
 687             }
 688             static T atomic_exchange( T volatile * pDest, T val ) CDS_NOEXCEPT
 689             {
 690                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 691             }
 692             static T atomic_exchange( T * pDest, T val ) CDS_NOEXCEPT
 693             {
 694                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 695             }
 696
 697             // cas
 698             static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 699             {
 700                 assert( expected );
 701                 return platform::cas32_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 702             }
 703             static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 704             {
 705                 assert( expected );
 706                 return platform::cas32_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 707             }
 708             static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 709             {
 710                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 711             }
 712             static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 713             {
 714                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 715             }
 716             static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 717             {
 718                 assert( expected );
 719                 return platform::cas32_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 720             }
 721             static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 722             {
 723                 assert( expected );
 724                 return platform::cas32_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 725             }
 726             static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 727             {
 728                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 729             }
 730             static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 731             {
 732                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 733             }
 734         };
 735
 736         template <typename T>
 737         struct atomic_integral_ops< T, 4 >
 738             : atomic_generic_ops< T, 4, T >
 739             , atomic_integral_bitwise_ops<T>
 740         {
 741             typedef atomic_integral_bitwise_ops<T> bitwise_ops;
 742             // fetch_add
 743             static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 744             {
 745 #           ifdef CDS_ATOMIC_fetch32_add_defined
 746                 return platform::fetch32_add( pDest, val, order );
 747 #           else
 748                 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
 749                 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
 750                 return cur;
 751 #           endif
 752             }
 753             static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 754             {
 755                 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 756             }
 757             static T atomic_fetch_add( T volatile * pDest, T val ) CDS_NOEXCEPT
 758             {
 759                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 760             }
 761             static T atomic_fetch_add( T * pDest, T val ) CDS_NOEXCEPT
 762             {
 763                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 764             }
 765
 766             // fetch_sub
 767             static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 768             {
 769 #           ifdef CDS_ATOMIC_fetch32_sub_defined
 770                 return platform::fetch32_sub( pDest, val, order );
 771 #           else
 772                 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
 773                 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
 774                 return cur;
 775 #           endif
 776             }
 777             static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 778             {
 779                 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 780             }
 781             static T atomic_fetch_sub( T volatile * pDest, T val ) CDS_NOEXCEPT
 782             {
 783                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
 784             }
 785             static T atomic_fetch_sub( T * pDest, T val ) CDS_NOEXCEPT
 786             {
 787                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
 788             }
 789
 790             // fetch_and
 791             static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 792             {
 793 #           ifdef CDS_ATOMIC_fetch32_and_defined
 794                 return platform::fetch32_and( pDest, val, order );
 795 #           else
 796                 return bitwise_ops::fetch_and( pDest, val, order );
 797 #           endif
 798             }
 799             static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 800             {
 801                 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 802             }
 803             static T atomic_fetch_and( T volatile * pDest, T val ) CDS_NOEXCEPT
 804             {
 805                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
 806             }
 807             static T atomic_fetch_and( T * pDest, T val ) CDS_NOEXCEPT
 808             {
 809                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
 810             }
 811
 812             // fetch_or
 813             static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 814             {
 815 #           ifdef CDS_ATOMIC_fetch32_or_defined
 816                 return platform::fetch32_or( pDest, val, order );
 817 #           else
 818                 return bitwise_ops::fetch_or( pDest, val, order );
 819 #           endif
 820             }
 821             static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 822             {
 823                 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 824             }
 825             static T atomic_fetch_or( T volatile * pDest, T val ) CDS_NOEXCEPT
 826             {
 827                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
 828             }
 829             static T atomic_fetch_or( T * pDest, T val ) CDS_NOEXCEPT
 830             {
 831                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
 832             }
 833
 834             // fetch_xor
 835             static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 836             {
 837 #           ifdef CDS_ATOMIC_fetch32_xor_defined
 838                 return platform::fetch32_xor( pDest, val, order );
 839 #           else
 840                 return bitwise_ops::fetch_xor( pDest, val, order );
 841 #           endif
 842             }
 843             static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 844             {
 845                 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 846             }
 847             static T atomic_fetch_xor( T volatile * pDest, T val ) CDS_NOEXCEPT
 848             {
 849                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
 850             }
 851             static T atomic_fetch_xor( T * pDest, T val ) CDS_NOEXCEPT
 852             {
 853                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
 854             }
 855         };
 856
 857
 858         // 64-bit atomic operations
 859
 860         template <typename T, typename Primary>
 861         struct atomic_generic_ops< T, 8, Primary >
 862         {
 863             typedef make_atomic_primary<T, Primary> primary;
 864
 865             // store
 866             static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
 867             {
 868                 platform::store64( primary::ptr(pDest), primary::val(v), order );
 869             }
 870             static void atomic_store_explicit( T * pDest, T v, memory_order order ) CDS_NOEXCEPT
 871             {
 872                 platform::store64( primary::ptr(pDest), primary::val(v), order );
 873             }
 874             static void atomic_store( T volatile * pDest, T v ) CDS_NOEXCEPT
 875             {
 876                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 877             }
 878             static void atomic_store( T * pDest, T v ) CDS_NOEXCEPT
 879             {
 880                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
 881             }
 882
 883             // load
 884             static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
 885             {
 886                 return primary::ret( platform::load64( primary::ptr(pSrc), order ));
 887             }
 888             static T atomic_load_explicit( T const * pSrc, memory_order order ) CDS_NOEXCEPT
 889             {
 890                 return primary::ret( platform::load64( primary::ptr(pSrc), order ));
 891             }
 892             static T atomic_load( T volatile const * pSrc ) CDS_NOEXCEPT
 893             {
 894                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 895             }
 896             static T atomic_load( T const * pSrc ) CDS_NOEXCEPT
 897             {
 898                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
 899             }
 900
 901             // exchange
 902             static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
 903             {
 904                 return primary::ret( platform::exchange64( primary::ptr(pDest), primary::val(val), order ));
 905             }
 906             static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) CDS_NOEXCEPT
 907             {
 908                 return primary::ret( platform::exchange64( primary::ptr(pDest), primary::val(val), order ));
 909             }
 910             static T atomic_exchange( T volatile * pDest, T val ) CDS_NOEXCEPT
 911             {
 912                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 913             }
 914             static T atomic_exchange( T * pDest, T val ) CDS_NOEXCEPT
 915             {
 916                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
 917             }
 918
 919             // cas
 920             static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 921             {
 922                 assert( expected );
 923                 return platform::cas64_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 924             }
 925             static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 926             {
 927                 assert( expected );
 928                 return platform::cas64_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 929             }
 930             static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 931             {
 932                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 933             }
 934             static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 935             {
 936                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 937             }
 938             static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 939             {
 940                 assert( expected );
 941                 return platform::cas64_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 942             }
 943             static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 944             {
 945                 assert( expected );
 946                 return platform::cas64_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
 947             }
 948             static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
 949             {
 950                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 951             }
 952             static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
 953             {
 954                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
 955             }
 956         };
 957
 958
 959         template <typename T>
 960         struct atomic_integral_ops< T, 8 >
 961             : atomic_generic_ops< T, 8, T >
 962             , atomic_integral_bitwise_ops<T>
 963         {
 964             typedef atomic_integral_bitwise_ops<T>  bitwise_ops;
 965             typedef atomic_generic_ops<T, 8, T>     general_ops;
 966
 967             // fetch_add
 968             static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 969             {
 970 #           ifdef CDS_ATOMIC_fetch64_add_defined
 971                 return platform::fetch64_add( pDest, val, order );
 972 #           else
 973                 T cur = general_ops::atomic_load_explicit( pDest, memory_order_relaxed );
 974                 do {} while ( !general_ops::atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
 975                 return cur;
 976 #           endif
 977             }
 978             static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
 979             {
 980                 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
 981             }
 982             static T atomic_fetch_add( T volatile * pDest, T val ) CDS_NOEXCEPT
 983             {
 984                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 985             }
 986             static T atomic_fetch_add( T * pDest, T val ) CDS_NOEXCEPT
 987             {
 988                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
 989             }
 990
 991             // fetch_sub
 992             static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 993             {
 994 #           ifdef CDS_ATOMIC_fetch64_sub_defined
 995                 return platform::fetch64_sub( pDest, val, order );
 996 #           else
 997                 T cur = general_ops::atomic_load_explicit( pDest, memory_order_relaxed );
 998                 do {} while ( !general_ops::atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
 999                 return cur;
1000 #           endif
1001             }
1002             static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
1003             {
1004                 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1005             }
1006             static T atomic_fetch_sub( T volatile * pDest, T val ) CDS_NOEXCEPT
1007             {
1008                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1009             }
1010             static T atomic_fetch_sub( T * pDest, T val ) CDS_NOEXCEPT
1011             {
1012                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1013             }
1014
1015             // fetch_and
1016             static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
1017             {
1018 #           ifdef CDS_ATOMIC_fetch64_and_defined
1019                 return platform::fetch64_and( pDest, val, order );
1020 #           else
1021                 return bitwise_ops::fetch_and( pDest, val, order );
1022 #           endif
1023             }
1024             static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
1025             {
1026                 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1027             }
1028             static T atomic_fetch_and( T volatile * pDest, T val ) CDS_NOEXCEPT
1029             {
1030                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
1031             }
1032             static T atomic_fetch_and( T * pDest, T val ) CDS_NOEXCEPT
1033             {
1034                 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
1035             }
1036
1037             // fetch_or
1038             static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
1039             {
1040 #           ifdef CDS_ATOMIC_fetch64_or_defined
1041                 return platform::fetch64_or( pDest, val, order );
1042 #           else
1043                 return bitwise_ops::fetch_or( pDest, val, order );
1044 #           endif
1045             }
1046             static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
1047             {
1048                 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1049             }
1050             static T atomic_fetch_or( T volatile * pDest, T val ) CDS_NOEXCEPT
1051             {
1052                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
1053             }
1054             static T atomic_fetch_or( T * pDest, T val ) CDS_NOEXCEPT
1055             {
1056                 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
1057             }
1058
1059             // fetch_xor
1060             static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
1061             {
1062 #           ifdef CDS_ATOMIC_fetch64_xor_defined
1063                 return platform::fetch64_xor( pDest, val, order );
1064 #           else
1065                 return bitwise_ops::fetch_xor( pDest, val, order );
1066 #           endif
1067             }
1068             static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
1069             {
1070                 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1071             }
1072             static T atomic_fetch_xor( T volatile * pDest, T val ) CDS_NOEXCEPT
1073             {
1074                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
1075             }
1076             static T atomic_fetch_xor( T * pDest, T val ) CDS_NOEXCEPT
1077             {
1078                 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
1079             }
1080         };
1081
1082
1083         // atomic pointer operations
1084         template <typename T>
1085         struct atomic_pointer_base
1086         {
1087             // store
1088             static void atomic_store_explicit( T * volatile * pDest, T * v, memory_order order ) CDS_NOEXCEPT
1089             {
1090                 platform::store_ptr( pDest, v, order );
1091             }
1092             static void atomic_store_explicit( T * * pDest, T * v, memory_order order ) CDS_NOEXCEPT
1093             {
1094                 platform::store_ptr( pDest, v, order );
1095             }
1096             static void atomic_store( T * volatile * pDest, T * v ) CDS_NOEXCEPT
1097             {
1098                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
1099             }
1100             static void atomic_store( T * * pDest, T * v ) CDS_NOEXCEPT
1101             {
1102                 atomic_store_explicit( pDest, v, memory_order_seq_cst );
1103             }
1104
1105             // load
1106             static T * atomic_load_explicit( T * volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
1107             {
1108                 return platform::load_ptr( pSrc, order );
1109             }
1110             static T * atomic_load_explicit( T * const * pSrc, memory_order order ) CDS_NOEXCEPT
1111             {
1112                 return platform::load_ptr( pSrc, order );
1113             }
1114             static T * atomic_load( T * volatile const * pSrc ) CDS_NOEXCEPT
1115             {
1116                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
1117             }
1118             static T * atomic_load( T * const * pSrc ) CDS_NOEXCEPT
1119             {
1120                 return atomic_load_explicit( pSrc, memory_order_seq_cst );
1121             }
1122
1123             // exchange
1124             static T * atomic_exchange_explicit( T * volatile * pDest, T * val, memory_order order ) CDS_NOEXCEPT
1125             {
1126                 return platform::exchange_ptr( pDest, val, order );
1127             }
1128             static T * atomic_exchange_explicit( T * * pDest, T * val, memory_order order ) CDS_NOEXCEPT
1129             {
1130                 return platform::exchange_ptr( pDest, val, order );
1131             }
1132             static T * atomic_exchange( T * volatile * pDest, T * val ) CDS_NOEXCEPT
1133             {
1134                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
1135             }
1136             static T * atomic_exchange( T * * pDest, T * val ) CDS_NOEXCEPT
1137             {
1138                 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
1139             }
1140
1141             // cas
1142             static bool atomic_compare_exchange_weak_explicit( T * volatile * pDest, T * * expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
1143             {
1144                 assert( expected );
1145                 return platform::cas_ptr_weak( pDest, *expected, desired, mo_success, mo_fail );
1146             }
1147             static bool atomic_compare_exchange_weak_explicit( T * * pDest, T * * expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
1148             {
1149                 assert( expected );
1150                 return platform::cas_ptr_weak( pDest, *expected, desired, mo_success, mo_fail );
1151             }
1152             static bool atomic_compare_exchange_weak( T * volatile * pDest, T ** expected, T * desired ) CDS_NOEXCEPT
1153             {
1154                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1155             }
1156             static bool atomic_compare_exchange_weak( T ** pDest, T ** expected, T * desired ) CDS_NOEXCEPT
1157             {
1158                 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1159             }
1160             static bool atomic_compare_exchange_strong_explicit( T * volatile * pDest, T ** expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
1161             {
1162                 assert( expected );
1163                 return platform::cas_ptr_strong( pDest, *expected, desired, mo_success, mo_fail );
1164             }
1165             static bool atomic_compare_exchange_strong_explicit( T ** pDest, T ** expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
1166             {
1167                 assert( expected );
1168                 return platform::cas_ptr_strong( pDest, *expected, desired, mo_success, mo_fail );
1169             }
1170             static bool atomic_compare_exchange_strong( T * volatile * pDest, T ** expected, T * desired ) CDS_NOEXCEPT
1171             {
1172                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1173             }
1174             static bool atomic_compare_exchange_strong( T ** pDest, T ** expected, T * desired ) CDS_NOEXCEPT
1175             {
1176                 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1177             }
1178         };
1179
1180         template <typename T>
1181         struct atomic_pointer: public atomic_pointer_base<T>
1182         {
1183             typedef atomic_pointer_base<T> base_class;
1184             // fetch_add
1185             static T * atomic_fetch_add_explicit(T * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
1186             {
1187 #           ifdef CDS_ATOMIC_fetch_ptr_add_defined
1188                 platform::fetch_ptr_add( pDest, val, order );
1189 #           else
1190                 T * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1191                 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
1192                 return cur;
1193 #           endif
1194             }
1195             static T * atomic_fetch_add_explicit(T * * pDest, ptrdiff_t val , memory_order order) CDS_NOEXCEPT
1196             {
1197                 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1198             }
1199             static T * atomic_fetch_add( T * volatile * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1200             {
1201                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1202             }
1203             static T * atomic_fetch_add( T ** pDest, ptrdiff_t val ) CDS_NOEXCEPT
1204             {
1205                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1206             }
1207
1208             // fetch_sub
1209             static T * atomic_fetch_sub_explicit(T * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
1210             {
1211 #           ifdef CDS_ATOMIC_fetch_ptr_sub_defined
1212                 platform::fetch_ptr_sub( pDest, val, order );
1213 #           else
1214                 T * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1215                 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
1216                 return cur;
1217 #           endif
1218             }
1219             static T * atomic_fetch_sub_explicit(T ** pDest, ptrdiff_t val , memory_order order) CDS_NOEXCEPT
1220             {
1221                 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1222             }
1223             static T * atomic_fetch_sub( T volatile * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1224             {
1225                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1226             }
1227             static T * atomic_fetch_sub( T * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1228             {
1229                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1230             }
1231         };
1232
1233         template <>
1234         struct atomic_pointer<void>: public atomic_pointer_base<void>
1235         {
1236             typedef atomic_pointer_base<void>   base_class;
1237
1238             // fetch_add
1239             static void * atomic_fetch_add_explicit(void * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
1240             {
1241                 void * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1242                 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, reinterpret_cast<char *>(cur) + val, order, memory_order_relaxed ));
1243                 return cur;
1244             }
1245             static void * atomic_fetch_add_explicit(void * * pDest, ptrdiff_t val , memory_order order) CDS_NOEXCEPT
1246             {
1247                 return atomic_fetch_add_explicit( reinterpret_cast<void * volatile *>( pDest ), val, order );
1248             }
1249             static void * atomic_fetch_add( void * volatile * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1250             {
1251                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1252             }
1253             static void * atomic_fetch_add( void ** pDest, ptrdiff_t val ) CDS_NOEXCEPT
1254             {
1255                 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1256             }
1257
1258             // fetch_sub
1259             static void * atomic_fetch_sub_explicit(void * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
1260             {
1261                 void * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1262                 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, reinterpret_cast<char *>(cur) - val, order, memory_order_relaxed ));
1263                 return cur;
1264             }
1265             static void * atomic_fetch_sub_explicit(void ** pDest, ptrdiff_t val , memory_order order) CDS_NOEXCEPT
1266             {
1267                 return atomic_fetch_sub_explicit( reinterpret_cast<void * volatile *>( pDest ), val, order );
1268             }
1269             static void * atomic_fetch_sub( void * volatile * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1270             {
1271                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1272             }
1273             static void * atomic_fetch_sub( void ** pDest, ptrdiff_t val ) CDS_NOEXCEPT
1274             {
1275                 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1276             }
1277         };
1278
1279         template <typename T>
1280         struct atomic_integral
1281         {
1282         private:
1283             typename cds::details::aligned_type<T, sizeof(T)>::type volatile m_val;
1284             //T volatile  m_val;
1285             typedef atomic_integral_ops<T, sizeof(T)>   atomic_ops;
1286         public:
1287             typedef T   atomic_type;
1288         public:
1289             bool is_lock_free() const volatile CDS_NOEXCEPT
1290             {
1291                 return true;
1292             }
1293             bool is_lock_free() const CDS_NOEXCEPT
1294             {
1295                 return true;
1296             }
1297             void store(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1298             {
1299                 atomic_ops::atomic_store_explicit( &m_val, val, order );
1300             }
1301             void store(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1302             {
1303                 atomic_ops::atomic_store_explicit( &m_val, val, order );
1304             }
1305
1306             T load(memory_order order = memory_order_seq_cst) const volatile CDS_NOEXCEPT
1307             {
1308                 return atomic_ops::atomic_load_explicit( &m_val, order );
1309             }
1310             T load(memory_order order  = memory_order_seq_cst) const CDS_NOEXCEPT
1311             {
1312                 return atomic_ops::atomic_load_explicit( &m_val, order );
1313             }
1314
1315             operator T() const volatile CDS_NOEXCEPT
1316             {
1317                 return load();
1318             }
1319             operator T() const CDS_NOEXCEPT
1320             {
1321                 return load();
1322             }
1323
1324             T exchange(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1325             {
1326                 return atomic_ops::atomic_exchange_explicit( &m_val, val, order );
1327             }
1328             T exchange(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1329             {
1330                 return atomic_ops::atomic_exchange_explicit( &m_val, val, order );
1331             }
1332
1333             bool compare_exchange_weak(T& expected, T desired , memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1334             {
1335                 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_val, &expected, desired, success_order, failure_order );
1336             }
1337             bool compare_exchange_weak(T& expected, T desired , memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1338             {
1339                 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_val, &expected, desired, success_order, failure_order );
1340             }
1341             bool compare_exchange_strong(T& expected, T desired , memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1342             {
1343                 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_val, &expected, desired, success_order, failure_order );
1344             }
1345             bool compare_exchange_strong(T& expected, T desired , memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1346             {
1347                 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_val, &expected, desired, success_order, failure_order );
1348             }
1349             bool compare_exchange_weak(T& expected, T desired , memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1350             {
1351                 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1352             }
1353             bool compare_exchange_weak(T& expected, T desired , memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1354             {
1355                 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1356             }
1357             bool compare_exchange_strong(T& expected, T desired , memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1358             {
1359                 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1360             }
1361             bool compare_exchange_strong(T& expected, T desired , memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1362             {
1363                 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1364             }
1365
1366             T fetch_add(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1367             {
1368                 return atomic_ops::atomic_fetch_add_explicit( &m_val, val, order );
1369             }
1370             T fetch_add(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1371             {
1372                 return atomic_ops::atomic_fetch_add_explicit( &m_val, val, order );
1373             }
1374             T fetch_sub(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1375             {
1376                 return atomic_ops::atomic_fetch_sub_explicit( &m_val, val, order );
1377             }
1378             T fetch_sub(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1379             {
1380                 return atomic_ops::atomic_fetch_sub_explicit( &m_val, val, order );
1381             }
1382             T fetch_and(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1383             {
1384                 return atomic_ops::atomic_fetch_and_explicit( &m_val, val, order );
1385             }
1386             T fetch_and(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1387             {
1388                 return atomic_ops::atomic_fetch_and_explicit( &m_val, val, order );
1389             }
1390
1391             T fetch_or(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1392             {
1393                 return atomic_ops::atomic_fetch_or_explicit( &m_val, val, order );
1394             }
1395             T fetch_or(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1396             {
1397                 return atomic_ops::atomic_fetch_or_explicit( &m_val, val, order );
1398             }
1399             T fetch_xor(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1400             {
1401                 return atomic_ops::atomic_fetch_xor_explicit( &m_val, val, order );
1402             }
1403             T fetch_xor(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1404             {
1405                 return atomic_ops::atomic_fetch_xor_explicit( &m_val, val, order );
1406             }
1407
1408             atomic_integral() = default;
1409             CDS_CONSTEXPR atomic_integral(T val) CDS_NOEXCEPT
1410                 : m_val(val)
1411                 {}
1412
1413             atomic_integral(const atomic_integral&) = delete;
1414             atomic_integral& operator=(const atomic_integral&) = delete;
1415             atomic_integral& operator=(const atomic_integral&) volatile = delete;
1416
1417             T operator=(T val) volatile CDS_NOEXCEPT
1418             {
1419                 store(val);
1420                 return val;
1421             }
1422             T operator=(T val) CDS_NOEXCEPT
1423             {
1424                 store(val);
1425                 return val;
1426             }
1427
1428             // Post inc/dec
1429             T operator++(int) volatile CDS_NOEXCEPT
1430             {
1431                 return fetch_add( 1 );
1432             }
1433             T operator++(int) CDS_NOEXCEPT
1434             {
1435                 return fetch_add( 1 );
1436             }
1437             T operator--(int) volatile CDS_NOEXCEPT
1438             {
1439                 return fetch_sub( 1 );
1440             }
1441             T operator--(int) CDS_NOEXCEPT
1442             {
1443                 return fetch_sub( 1 );
1444             }
1445
1446             // Pre inc/dec
1447             T operator++() volatile CDS_NOEXCEPT
1448             {
1449                 return fetch_add( 1 ) + 1;
1450             }
1451             T operator++() CDS_NOEXCEPT
1452             {
1453                 return fetch_add( 1 ) + 1;
1454             }
1455             T operator--() volatile CDS_NOEXCEPT
1456             {
1457                 return fetch_sub( 1 ) - 1;
1458             }
1459             T operator--() CDS_NOEXCEPT
1460             {
1461                 return fetch_sub( 1 ) - 1;
1462             }
1463
1464             // op=
1465             T operator+=(T val) volatile CDS_NOEXCEPT
1466             {
1467                 return fetch_add( val ) + val;
1468             }
1469             T operator+=(T val) CDS_NOEXCEPT
1470             {
1471                 return fetch_add( val ) + val;
1472             }
1473             T operator-=(T val) volatile CDS_NOEXCEPT
1474             {
1475                 return fetch_sub( val ) - val;
1476             }
1477             T operator-=(T val) CDS_NOEXCEPT
1478             {
1479                 return fetch_sub( val ) - val;
1480             }
1481             T operator&=(T val) volatile CDS_NOEXCEPT
1482             {
1483                 return fetch_and( val ) & val;
1484             }
1485             T operator&=(T val) CDS_NOEXCEPT
1486             {
1487                 return fetch_and( val ) & val;
1488             }
1489             T operator|=(T val) volatile CDS_NOEXCEPT
1490             {
1491                 return fetch_or( val ) | val;
1492             }
1493             T operator|=(T val) CDS_NOEXCEPT
1494             {
1495                 return fetch_or( val ) | val;
1496             }
1497             T operator^=(T val) volatile CDS_NOEXCEPT
1498             {
1499                 return fetch_xor( val ) ^ val;
1500             }
1501             T operator^=(T val) CDS_NOEXCEPT
1502             {
1503                 return fetch_xor( val ) ^ val;
1504             }
1505         };
1506
1507         template <typename Type>
1508         struct select_primary_type {
1509             typedef typename details::primary_type<sizeof(Type)>::type type;
1510         };
1511         template <>
1512         struct select_primary_type<bool> {
1513             typedef bool type;
1514         };
1515
1516     }   // namespace details
1517
1518     template <class T>
1519     struct atomic
1520     {
1521     private:
1522         typedef details::atomic_generic_ops<T, sizeof(T), typename details::select_primary_type<T>::type >  atomic_ops;
1523
1524         T volatile m_data;
1525     public:
1526         bool is_lock_free() const volatile CDS_NOEXCEPT
1527         {
1528             return true;
1529         }
1530         bool is_lock_free() const CDS_NOEXCEPT
1531         {
1532             return true;
1533         }
1534
1535         void store(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1536         {
1537             atomic_ops::atomic_store_explicit( &m_data, val, order );
1538         }
1539         void store(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1540         {
1541             atomic_ops::atomic_store_explicit( &m_data, val, order );
1542         }
1543
1544         T load(memory_order order = memory_order_seq_cst) const volatile CDS_NOEXCEPT
1545         {
1546             return atomic_ops::atomic_load_explicit( &m_data, order );
1547         }
1548         T load(memory_order order = memory_order_seq_cst) const CDS_NOEXCEPT
1549         {
1550            return atomic_ops::atomic_load_explicit( &m_data, order );
1551         }
1552
1553         operator T() const volatile CDS_NOEXCEPT
1554         {
1555             return load();
1556         }
1557         operator T() const CDS_NOEXCEPT
1558         {
1559             return load();
1560         }
1561
1562         T exchange(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1563         {
1564             return atomic_ops::atomic_exchange_explicit( &m_data, val, order );
1565         }
1566         T exchange(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1567         {
1568             return atomic_ops::atomic_exchange_explicit( &m_data, val, order );
1569         }
1570
1571         bool compare_exchange_weak(T& expected, T desired, memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1572         {
1573             return atomic_ops::atomic_compare_exchange_weak_explicit( &m_data, &expected, desired, success_order, failure_order );
1574         }
1575         bool compare_exchange_weak(T& expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1576         {
1577             return atomic_ops::atomic_compare_exchange_weak_explicit( &m_data, &expected, desired, success_order, failure_order );
1578         }
1579         bool compare_exchange_strong(T& expected, T desired, memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1580         {
1581             return atomic_ops::atomic_compare_exchange_strong_explicit( &m_data, &expected, desired, success_order, failure_order );
1582         }
1583         bool compare_exchange_strong(T& expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1584         {
1585             return atomic_ops::atomic_compare_exchange_strong_explicit( &m_data, &expected, desired, success_order, failure_order );
1586         }
1587         bool compare_exchange_weak(T& expected, T desired, memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1588         {
1589             return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1590         }
1591         bool compare_exchange_weak(T& expected, T desired, memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1592         {
1593             return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1594         }
1595         bool compare_exchange_strong(T& expected, T desired, memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1596         {
1597             return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1598         }
1599         bool compare_exchange_strong(T& expected, T desired, memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1600         {
1601             return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1602         }
1603
1604         atomic() = default;
1605         CDS_CONSTEXPR atomic(T val)
1606             : m_data( val )
1607             {}
1608
1609         atomic(const atomic&) = delete;
1610         atomic& operator=(const atomic&) = delete;
1611         atomic& operator=(const atomic&) volatile = delete;
1612
1613         T operator=(T val) volatile CDS_NOEXCEPT
1614         {
1615             store( val );
1616             return val;
1617         }
1618         T operator=(T val) CDS_NOEXCEPT
1619         {
1620             store( val );
1621             return val;
1622         }
1623     };
1624
1625 #   define CDS_DECLARE_ATOMIC_INTEGRAL( _type ) \
1626     template <> \
1627     struct atomic<_type>: public details::atomic_integral<_type> \
1628     { \
1629     private: \
1630         typedef details::atomic_integral<_type>   base_class  ; \
1631     public: \
1632         atomic() = default; \
1633         atomic(_type val) CDS_NOEXCEPT : base_class(val) {} \
1634         atomic(const atomic&) = delete; \
1635         atomic& operator=(const atomic&) = delete; \
1636         atomic& operator=(const atomic&) volatile = delete; \
1637         _type operator=(_type val) volatile CDS_NOEXCEPT { return base_class::operator=(val); } \
1638         _type operator=(_type val) CDS_NOEXCEPT { return base_class::operator=(val); } \
1639     };
1640
1641     CDS_DECLARE_ATOMIC_INTEGRAL(char)
1642     CDS_DECLARE_ATOMIC_INTEGRAL(signed char)
1643     CDS_DECLARE_ATOMIC_INTEGRAL(unsigned char)
1644     CDS_DECLARE_ATOMIC_INTEGRAL(short)
1645     CDS_DECLARE_ATOMIC_INTEGRAL(unsigned short)
1646     CDS_DECLARE_ATOMIC_INTEGRAL(int)
1647     CDS_DECLARE_ATOMIC_INTEGRAL(unsigned int)
1648     CDS_DECLARE_ATOMIC_INTEGRAL(long)
1649     CDS_DECLARE_ATOMIC_INTEGRAL(unsigned long)
1650     CDS_DECLARE_ATOMIC_INTEGRAL(long long)
1651     CDS_DECLARE_ATOMIC_INTEGRAL(unsigned long long)
1652 //#if CDS_COMPILER == CDS_COMPILER_GCC && CDS_COMPILER_VERSION >= 40400
1653 //    CDS_DECLARE_ATOMIC_INTEGRAL(char16_t)
1654 //    CDS_DECLARE_ATOMIC_INTEGRAL(char32_t)
1655 //#endif
1656 //    CDS_DECLARE_ATOMIC_INTEGRAL(wchar_t)
1657
1658 #   undef CDS_DECLARE_ATOMIC_INTEGRAL
1659
1660
1661     template <typename T>
1662     class atomic<T *>
1663     {
1664     private:
1665         T * volatile m_ptr;
1666         typedef details::atomic_pointer<T>  atomic_ops;
1667     public:
1668         bool is_lock_free() const volatile CDS_NOEXCEPT
1669         {
1670             return true;
1671         }
1672         bool is_lock_free() const CDS_NOEXCEPT
1673         {
1674             return true;
1675         }
1676
1677         void store(T * val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1678         {
1679             atomic_ops::atomic_store_explicit( &m_ptr, val, order );
1680         }
1681         void store(T * val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1682         {
1683             atomic_ops::atomic_store_explicit( &m_ptr, val, order );
1684         }
1685
1686         T * load(memory_order order = memory_order_seq_cst) const volatile CDS_NOEXCEPT
1687         {
1688             return atomic_ops::atomic_load_explicit( &m_ptr, order );
1689         }
1690         T * load(memory_order order = memory_order_seq_cst) const CDS_NOEXCEPT
1691         {
1692             return atomic_ops::atomic_load_explicit( &m_ptr, order );
1693         }
1694
1695         operator T *() const volatile CDS_NOEXCEPT
1696         {
1697             return load();
1698         }
1699         operator T *() const CDS_NOEXCEPT
1700         {
1701             return load();
1702         }
1703
1704         T * exchange(T * val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1705         {
1706             return atomic_ops::atomic_exchange_explicit( &m_ptr, val, order );
1707         }
1708         T * exchange(T * val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1709         {
1710             return atomic_ops::atomic_exchange_explicit( &m_ptr, val, order );
1711         }
1712
1713         bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1714         {
1715             return atomic_ops::atomic_compare_exchange_weak_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1716         }
1717         bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1718         {
1719             return atomic_ops::atomic_compare_exchange_weak_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1720         }
1721         bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1722         {
1723             return atomic_ops::atomic_compare_exchange_strong_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1724         }
1725         bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1726         {
1727             return atomic_ops::atomic_compare_exchange_strong_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1728         }
1729         bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1730         {
1731             return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1732         }
1733         bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1734         {
1735             return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1736         }
1737         bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1738         {
1739             return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1740         }
1741         bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1742         {
1743             return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1744         }
1745
1746         T * fetch_add(ptrdiff_t offset, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1747         {
1748             return atomic_ops::atomic_fetch_add_explicit( &m_ptr, offset, order );
1749         }
1750         T * fetch_add(ptrdiff_t offset, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1751         {
1752             return atomic_ops::atomic_fetch_add_explicit( &m_ptr, offset, order );
1753         }
1754
1755         T * fetch_sub(ptrdiff_t offset, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1756         {
1757             return atomic_ops::atomic_fetch_sub_explicit( &m_ptr, offset, order );
1758         }
1759         T * fetch_sub(ptrdiff_t offset, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1760         {
1761             return atomic_ops::atomic_fetch_sub_explicit( &m_ptr, offset, order );
1762         }
1763
1764         atomic() = default;
1765         CDS_CONSTEXPR atomic(T * val) CDS_NOEXCEPT
1766             : m_ptr( val )
1767         {}
1768
1769         atomic(const atomic&) = delete;
1770         atomic& operator=(const atomic&) = delete;
1771         atomic& operator=(const atomic&) volatile = delete;
1772
1773         T * operator=(T * val) volatile CDS_NOEXCEPT
1774         {
1775             store( val );
1776             return val;
1777         }
1778         T * operator=(T * val) CDS_NOEXCEPT
1779         {
1780             store( val );
1781             return val;
1782         }
1783     };
1784
1785     // Atomic typedefs
1786     typedef atomic<bool>            atomic_bool;
1787     typedef atomic<char>            atomic_char;
1788     typedef atomic<signed char>     atomic_schar;
1789     typedef atomic<unsigned char>   atomic_uchar;
1790     typedef atomic<short>           atomic_short;
1791     typedef atomic<unsigned short>  atomic_ushort;
1792     typedef atomic<int>             atomic_int;
1793     typedef atomic<unsigned int>    atomic_uint;
1794     typedef atomic<long>            atomic_long;
1795     typedef atomic<unsigned long>   atomic_ulong;
1796     typedef atomic<long long>       atomic_llong;
1797     typedef atomic<unsigned long long> atomic_ullong;
1798 #if ( CDS_COMPILER == CDS_COMPILER_GCC && CDS_COMPILER_VERSION >= 40400 ) || CDS_COMPILER == CDS_COMPILER_CLANG
1799     typedef atomic<char16_t>        atomic_char16_t;
1800     typedef atomic<char32_t>        atomic_char32_t;
1801 #endif
1802     typedef atomic<wchar_t>         atomic_wchar_t;
1803
1804
1805     typedef atomic<cds::int_least8_t>    atomic_int_least8_t;
1806     typedef atomic<cds::uint_least8_t>   atomic_uint_least8_t;
1807     typedef atomic<cds::int_least16_t>   atomic_int_least16_t;
1808     typedef atomic<cds::uint_least16_t>  atomic_uint_least16_t;
1809     typedef atomic<cds::int_least32_t>   atomic_int_least32_t;
1810     typedef atomic<cds::uint_least32_t>  atomic_uint_least32_t;
1811     typedef atomic<cds::int_least64_t>   atomic_int_least64_t;
1812     typedef atomic<cds::uint_least64_t>  atomic_uint_least64_t;
1813     typedef atomic<cds::int_fast8_t>     atomic_int_fast8_t;
1814     typedef atomic<cds::uint_fast8_t>    atomic_uint_fast8_t;
1815     typedef atomic<cds::int_fast16_t>    atomic_int_fast16_t;
1816     typedef atomic<cds::uint_fast16_t>   atomic_uint_fast16_t;
1817     typedef atomic<cds::int_fast32_t>    atomic_int_fast32_t;
1818     typedef atomic<cds::uint_fast32_t>   atomic_uint_fast32_t;
1819     typedef atomic<cds::int_fast64_t>    atomic_int_fast64_t;
1820     typedef atomic<cds::uint_fast64_t>   atomic_uint_fast64_t;
1821     typedef atomic<intptr_t>             atomic_intptr_t;
1822     typedef atomic<uintptr_t>            atomic_uintptr_t;
1823     typedef atomic<size_t>               atomic_size_t;
1824     typedef atomic<ptrdiff_t>            atomic_ptrdiff_t;
1825     typedef atomic<cds::intmax_t>        atomic_intmax_t;
1826     typedef atomic<cds::uintmax_t>       atomic_uintmax_t;
1827
1828     template <class T>
1829     static inline bool atomic_is_lock_free(const volatile atomic<T> * p) CDS_NOEXCEPT
1830     {
1831         return p->is_lock_free();
1832     }
1833
1834     template <class T>
1835     static inline bool atomic_is_lock_free(const atomic<T> * p ) CDS_NOEXCEPT
1836     {
1837         return p->is_lock_free();
1838     }
1839
1840     /*
1841     template <class T>
1842     static inline void atomic_init(volatile atomic<T> * p, T val) CDS_NOEXCEPT
1843     {
1844         p->init( val );
1845     }
1846
1847     template <class T>
1848     static inline void atomic_init( atomic<T> * p, T val) CDS_NOEXCEPT
1849     {
1850         p->init( val );
1851     }
1852     */
1853
1854     template <class T>
1855     static inline void atomic_store(volatile atomic<T>* p, T val) CDS_NOEXCEPT
1856     {
1857         p->store(val);
1858     }
1859     template <class T>
1860     static inline void atomic_store(atomic<T>* p, T val) CDS_NOEXCEPT
1861     {
1862         p->store( val );
1863     }
1864
1865     template <class T>
1866     static inline void atomic_store_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1867     {
1868         p->store( val, order );
1869     }
1870     template <class T>
1871     static inline void atomic_store_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1872     {
1873         p->store( val, order );
1874     }
1875
1876     template <class T>
1877     static inline T atomic_load(const volatile atomic<T>* p) CDS_NOEXCEPT
1878     {
1879         return p->load();
1880     }
1881     template <class T>
1882     static inline T atomic_load(const atomic<T>* p) CDS_NOEXCEPT
1883     {
1884         return p->load();
1885     }
1886
1887     template <class T>
1888     static inline T atomic_load_explicit(const volatile atomic<T>* p, memory_order order) CDS_NOEXCEPT
1889     {
1890         return p->load( order );
1891     }
1892     template <class T>
1893     static inline T atomic_load_explicit(const atomic<T>* p, memory_order order) CDS_NOEXCEPT
1894     {
1895         return p->load( order );
1896     }
1897
1898     template <class T>
1899     static inline T atomic_exchange(volatile atomic<T>* p, T val) CDS_NOEXCEPT
1900     {
1901         return p->exchange( val );
1902     }
1903     template <class T>
1904     static inline T atomic_exchange(atomic<T>* p, T val ) CDS_NOEXCEPT
1905     {
1906         return p->exchange( val );
1907     }
1908
1909     template <class T>
1910     static inline T atomic_exchange_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1911     {
1912         return p->exchange( val, order );
1913     }
1914     template <class T>
1915     static inline T atomic_exchange_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1916     {
1917         return p->exchange( val, order );
1918     }
1919
1920     template <class T>
1921     static inline bool atomic_compare_exchange_weak(volatile atomic<T>* p, T* expected, T desired) CDS_NOEXCEPT
1922     {
1923         return p->compare_exchange_weak( *expected, desired );
1924     }
1925     template <class T>
1926     static inline bool atomic_compare_exchange_weak(atomic<T>* p, T* expected, T desired) CDS_NOEXCEPT
1927     {
1928         return p->compare_exchange_weak( *expected, desired );
1929     }
1930
1931     template <class T>
1932     static inline bool atomic_compare_exchange_strong(volatile atomic<T>* p, T* expected, T desired) CDS_NOEXCEPT
1933     {
1934         return p->compare_exchange_strong( *expected, desired );
1935     }
1936     template <class T>
1937     static inline bool atomic_compare_exchange_strong(atomic<T>* p, T* expected, T desired) CDS_NOEXCEPT
1938     {
1939         return p->compare_exchange_strong( *expected, desired );
1940     }
1941
1942     template <class T>
1943     static inline bool atomic_compare_exchange_weak_explicit(volatile atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1944     {
1945         return p->compare_exchange_weak( *expected, desired, success_order, failure_order );
1946     }
1947     template <class T>
1948     static inline bool atomic_compare_exchange_weak_explicit(atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1949     {
1950         return p->compare_exchange_weak( *expected, desired, success_order, failure_order );
1951     }
1952
1953     template <class T>
1954     static inline bool atomic_compare_exchange_strong_explicit(volatile atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1955     {
1956         return p->compare_exchange_strong( *expected, desired, success_order, failure_order );
1957     }
1958     template <class T>
1959     static inline bool atomic_compare_exchange_strong_explicit(atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1960     {
1961         return p->compare_exchange_strong( *expected, desired, success_order, failure_order );
1962     }
1963
1964     template <class T>
1965     static inline T atomic_fetch_add(volatile atomic<T>* p, T val) CDS_NOEXCEPT
1966     {
1967         return p->fetch_add( val );
1968     }
1969     template <class T>
1970     static inline T atomic_fetch_add(atomic<T>* p, T val) CDS_NOEXCEPT
1971     {
1972         return p->fetch_add( val );
1973     }
1974     template <class T>
1975     static inline T * atomic_fetch_add(volatile atomic<T *>* p, ptrdiff_t offset) CDS_NOEXCEPT
1976     {
1977         return p->fetch_add( offset );
1978     }
1979     template <class T>
1980     static inline T * atomic_fetch_add(atomic<T *>* p, ptrdiff_t offset) CDS_NOEXCEPT
1981     {
1982         return p->fetch_add( offset );
1983     }
1984
1985     template <class T>
1986     static inline T atomic_fetch_add_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1987     {
1988         return p->fetch_add( val, order );
1989     }
1990     template <class T>
1991     static inline T atomic_fetch_add_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1992     {
1993         return p->fetch_add( val, order );
1994     }
1995     template <class T>
1996     static inline T * atomic_fetch_add_explicit(volatile atomic<T *>* p, ptrdiff_t offset, memory_order order) CDS_NOEXCEPT
1997     {
1998         return p->fetch_add( offset, order );
1999     }
2000     template <class T>
2001     static inline T * atomic_fetch_add_explicit(atomic<T *>* p, ptrdiff_t offset, memory_order order) CDS_NOEXCEPT
2002     {
2003         return p->fetch_add( offset, order );
2004     }
2005
2006     template <class T>
2007     static inline T atomic_fetch_sub(volatile atomic<T>* p, T val) CDS_NOEXCEPT
2008     {
2009         return p->fetch_sub( val );
2010     }
2011     template <class T>
2012     static inline T atomic_fetch_sub(atomic<T>* p, T val) CDS_NOEXCEPT
2013     {
2014         return p->fetch_sub( val );
2015     }
2016     template <class T>
2017     static inline T * atomic_fetch_sub(volatile atomic<T *>* p, ptrdiff_t offset) CDS_NOEXCEPT
2018     {
2019         return p->fetch_sub( offset );
2020     }
2021     template <class T>
2022     static inline T * atomic_fetch_sub(atomic<T *>* p, ptrdiff_t offset) CDS_NOEXCEPT
2023     {
2024         return p->fetch_sub( offset );
2025     }
2026
2027     template <class T>
2028     static inline T atomic_fetch_sub_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2029     {
2030         return p->fetch_sub( val, order );
2031     }
2032     template <class T>
2033     static inline T atomic_fetch_sub_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2034     {
2035         return p->fetch_sub( val, order );
2036     }
2037     template <class T>
2038     static inline T * atomic_fetch_sub_explicit(volatile atomic<T *>* p, ptrdiff_t offset, memory_order order) CDS_NOEXCEPT
2039     {
2040         return p->fetch_sub( offset, order );
2041     }
2042     template <class T>
2043     static inline T * atomic_fetch_sub_explicit(atomic<T *>* p, ptrdiff_t offset, memory_order order) CDS_NOEXCEPT
2044     {
2045         return p->fetch_sub( offset, order );
2046     }
2047
2048     template <class T>
2049     static inline T atomic_fetch_and(volatile atomic<T>* p, T val) CDS_NOEXCEPT
2050     {
2051         return p->fetch_and( val );
2052     }
2053     template <class T>
2054     static inline T atomic_fetch_and(atomic<T>* p, T val) CDS_NOEXCEPT
2055     {
2056         return p->fetch_and( val );
2057     }
2058
2059     template <class T>
2060     static inline T atomic_fetch_and_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2061     {
2062         return p->fetch_and( val, order );
2063     }
2064     template <class T>
2065     static inline T atomic_fetch_and_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2066     {
2067         return p->fetch_and( val, order );
2068     }
2069
2070     template <class T>
2071     static inline T atomic_fetch_or(volatile atomic<T>* p, T val) CDS_NOEXCEPT
2072     {
2073         return p->fetch_or( val );
2074     }
2075     template <class T>
2076     static inline T atomic_fetch_or(atomic<T>* p, T val) CDS_NOEXCEPT
2077     {
2078         return p->fetch_or( val );
2079     }
2080
2081     template <class T>
2082     static inline T atomic_fetch_or_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2083     {
2084         return p->fetch_or( val, order );
2085     }
2086     template <class T>
2087     static inline T atomic_fetch_or_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2088     {
2089         return p->fetch_or( val, order );
2090     }
2091
2092     template <class T>
2093     static inline T atomic_fetch_xor(volatile atomic<T>* p, T val) CDS_NOEXCEPT
2094     {
2095         return p->fetch_xor( val );
2096     }
2097     template <class T>
2098     static inline T atomic_fetch_xor(atomic<T>* p, T val) CDS_NOEXCEPT
2099     {
2100         return p->fetch_xor( val );
2101     }
2102
2103     template <class T>
2104     static inline T atomic_fetch_xor_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2105     {
2106         return p->fetch_xor( val, order );
2107     }
2108     template <class T>
2109     static inline T atomic_fetch_xor_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2110     {
2111         return p->fetch_xor( val, order );
2112     }
2113
2114     // Atomic flag type
2115     typedef struct atomic_flag
2116     {
2117         void clear( memory_order order = memory_order_seq_cst ) volatile CDS_NOEXCEPT
2118         {
2119             assert( order != memory_order_acquire
2120                 && order != memory_order_acq_rel
2121                 && order != memory_order_consume
2122                 );
2123             platform::atomic_flag_clear( &m_Flag, order );
2124         }
2125         void clear( memory_order order = memory_order_seq_cst ) CDS_NOEXCEPT
2126         {
2127             assert( order != memory_order_acquire
2128                 && order != memory_order_acq_rel
2129                 && order != memory_order_consume
2130                 );
2131             platform::atomic_flag_clear( &m_Flag, order );
2132         }
2133
2134         bool test_and_set( memory_order order = memory_order_seq_cst ) volatile CDS_NOEXCEPT
2135         {
2136             return platform::atomic_flag_tas( &m_Flag, order );
2137         }
2138         bool test_and_set( memory_order order = memory_order_seq_cst ) CDS_NOEXCEPT
2139         {
2140             return platform::atomic_flag_tas( &m_Flag, order );
2141         }
2142
2143         atomic_flag() = default;
2144
2145         atomic_flag(const atomic_flag&) = delete;
2146         atomic_flag& operator=(const atomic_flag&) = delete;
2147         atomic_flag& operator=(const atomic_flag&) volatile = delete;
2148
2149         platform::atomic_flag_type volatile m_Flag;
2150     } atomic_flag;
2151
2152     static inline bool atomic_flag_test_and_set(volatile atomic_flag* p) CDS_NOEXCEPT
2153     {
2154         return p->test_and_set();
2155     }
2156     static inline bool atomic_flag_test_and_set(atomic_flag * p) CDS_NOEXCEPT
2157     {
2158         return p->test_and_set();
2159     }
2160     static inline bool atomic_flag_test_and_set_explicit(volatile atomic_flag* p, memory_order order) CDS_NOEXCEPT
2161     {
2162         return p->test_and_set( order );
2163     }
2164     static inline bool atomic_flag_test_and_set_explicit(atomic_flag* p, memory_order order) CDS_NOEXCEPT
2165     {
2166         return p->test_and_set( order );
2167     }
2168     static inline void atomic_flag_clear(volatile atomic_flag* p) CDS_NOEXCEPT
2169     {
2170         return p->clear();
2171     }
2172     static inline void atomic_flag_clear(atomic_flag* p) CDS_NOEXCEPT
2173     {
2174         return p->clear();
2175     }
2176     static inline void atomic_flag_clear_explicit(volatile atomic_flag* p, memory_order order) CDS_NOEXCEPT
2177     {
2178         return p->clear( order );
2179     }
2180     static inline void atomic_flag_clear_explicit(atomic_flag* p, memory_order order) CDS_NOEXCEPT
2181     {
2182         return p->clear( order );
2183     }
2184
2185     // Fences
2186     static inline void atomic_thread_fence(memory_order order) CDS_NOEXCEPT
2187     {
2188         platform::thread_fence( order );
2189         CDS_COMPILER_RW_BARRIER;
2190     }
2191     static inline void atomic_signal_fence(memory_order order) CDS_NOEXCEPT
2192     {
2193         platform::signal_fence( order );
2194     }
2195
2196 }}  // namespace cds::cxx11_atomic
2197
2198 //@endcond
2199 #endif // #ifndef __CDS_COMPILER_CXX11_ATOMIC_H