cds/compiler/vc/amd64/cxx11_atomic.h

   1 //$$CDS-header$$
   2
   3 #ifndef __CDS_COMPILER_VC_AMD64_CXX11_ATOMIC_H
   4 #define __CDS_COMPILER_VC_AMD64_CXX11_ATOMIC_H
   5
   6 #include <intrin.h>
   7 #include <emmintrin.h>  // for 128bit atomic load/store
   8 #include <cds/details/is_aligned.h>
   9
  10 #pragma intrinsic( _InterlockedIncrement )
  11 #pragma intrinsic( _InterlockedDecrement )
  12 #pragma intrinsic( _InterlockedCompareExchange )
  13 #pragma intrinsic( _InterlockedCompareExchangePointer )
  14 #pragma intrinsic( _InterlockedCompareExchange16 )
  15 #pragma intrinsic( _InterlockedCompareExchange64 )
  16 #pragma intrinsic( _InterlockedExchange )
  17 #pragma intrinsic( _InterlockedExchange64 )
  18 #pragma intrinsic( _InterlockedExchangePointer )
  19 #pragma intrinsic( _InterlockedExchangeAdd )
  20 #pragma intrinsic( _InterlockedExchangeAdd64 )
  21 //#pragma intrinsic( _InterlockedAnd )
  22 //#pragma intrinsic( _InterlockedOr )
  23 //#pragma intrinsic( _InterlockedXor )
  24 //#pragma intrinsic( _InterlockedAnd64 )
  25 //#pragma intrinsic( _InterlockedOr64 )
  26 //#pragma intrinsic( _InterlockedXor64 )
  27 #pragma intrinsic( _interlockedbittestandset )
  28 #if _MSC_VER >= 1600
  29 #   pragma intrinsic( _InterlockedCompareExchange8 )
  30 #   pragma intrinsic( _InterlockedExchange8 )
  31 #   pragma intrinsic( _InterlockedExchange16 )
  32 #endif
  33
  34 //@cond
  35 namespace cds { namespace cxx11_atomics {
  36     namespace platform { CDS_CXX11_INLINE_NAMESPACE namespace vc { CDS_CXX11_INLINE_NAMESPACE namespace amd64 {
  37
  38         static inline void fence_before( memory_order order ) CDS_NOEXCEPT
  39         {
  40             switch(order) {
  41             case memory_order_relaxed:
  42             case memory_order_acquire:
  43             case memory_order_consume:
  44                 break;
  45             case memory_order_release:
  46             case memory_order_acq_rel:
  47                 CDS_COMPILER_RW_BARRIER;
  48                 break;
  49             case memory_order_seq_cst:
  50                 CDS_COMPILER_RW_BARRIER;
  51                 break;
  52             }
  53         }
  54
  55         static inline void fence_after( memory_order order ) CDS_NOEXCEPT
  56         {
  57             switch(order) {
  58             case memory_order_acquire:
  59             case memory_order_acq_rel:
  60                 CDS_COMPILER_RW_BARRIER;
  61                 break;
  62             case memory_order_relaxed:
  63             case memory_order_consume:
  64             case memory_order_release:
  65                 break;
  66             case memory_order_seq_cst:
  67                 CDS_COMPILER_RW_BARRIER;
  68                 break;
  69             }
  70         }
  71
  72         static inline void full_fence()
  73         {
  74             // MS VC does not support inline assembler in C code.
  75             // So, we use InterlockedExchange for full fence instead of mfence inst
  76             long t;
  77             _InterlockedExchange( &t, 0 );
  78         }
  79
  80         static inline void fence_after_load(memory_order order) CDS_NOEXCEPT
  81         {
  82             switch(order) {
  83             case memory_order_relaxed:
  84             case memory_order_release:
  85                 break;
  86             case memory_order_acquire:
  87             case memory_order_acq_rel:
  88                 CDS_COMPILER_RW_BARRIER;
  89                 break;
  90             case memory_order_consume:
  91                 break;
  92             case memory_order_seq_cst:
  93                 full_fence();
  94                 break;
  95             default:;
  96             }
  97         }
  98
  99         //-----------------------------------------------------------------------------
 100         // fences
 101         //-----------------------------------------------------------------------------
 102         static inline void thread_fence(memory_order order) CDS_NOEXCEPT
 103         {
 104             switch(order)
 105             {
 106                 case memory_order_relaxed:
 107                 case memory_order_consume:
 108                     break;
 109                 case memory_order_release:
 110                 case memory_order_acquire:
 111                 case memory_order_acq_rel:
 112                     CDS_COMPILER_RW_BARRIER;
 113                     break;
 114                 case memory_order_seq_cst:
 115                     full_fence();
 116                     break;
 117                 default:;
 118             }
 119         }
 120
 121         static inline void signal_fence(memory_order order) CDS_NOEXCEPT
 122         {
 123             // C++11: 29.8.8: only compiler optimization, no hardware instructions
 124             switch(order)
 125             {
 126                 case memory_order_relaxed:
 127                     break;
 128                 case memory_order_consume:
 129                 case memory_order_release:
 130                 case memory_order_acquire:
 131                 case memory_order_acq_rel:
 132                 case memory_order_seq_cst:
 133                     CDS_COMPILER_RW_BARRIER;
 134                     break;
 135                 default:;
 136             }
 137         }
 138
 139         //-----------------------------------------------------------------------------
 140         // atomic flag primitives
 141         //-----------------------------------------------------------------------------
 142
 143         typedef unsigned char atomic_flag_type;
 144         static inline bool atomic_flag_tas( atomic_flag_type volatile * pFlag, memory_order /*order*/ ) CDS_NOEXCEPT
 145         {
 146             return _interlockedbittestandset( (long volatile *) pFlag, 0 ) != 0;
 147         }
 148
 149         static inline void atomic_flag_clear( atomic_flag_type volatile * pFlag, memory_order order ) CDS_NOEXCEPT
 150         {
 151             assert( order != memory_order_acquire
 152                 && order != memory_order_acq_rel
 153                 );
 154
 155             fence_before( order );
 156             *pFlag = 0;
 157             fence_after( order );
 158         }
 159
 160         //-----------------------------------------------------------------------------
 161         // 8bit primitives
 162         //-----------------------------------------------------------------------------
 163
 164 #if _MSC_VER >= 1600
 165 #   pragma warning(push)
 166         // Disable warning C4800: 'char' : forcing value to bool 'true' or 'false' (performance warning)
 167 #   pragma warning( disable: 4800 )
 168 #endif
 169         template <typename T>
 170         static inline bool cas8_strong( T volatile * pDest, T& expected, T desired, memory_order /*mo_success*/, memory_order /*mo_fail*/ ) CDS_NOEXCEPT
 171         {
 172             static_assert( sizeof(T) == 1, "Illegal size of operand" );
 173
 174 #       if _MSC_VER >= 1600
 175             // VC 2010 +
 176             T prev = expected;
 177             expected = (T) _InterlockedCompareExchange8( (char volatile*) pDest, (char) desired, (char) expected );
 178             return expected == prev;
 179 #       else
 180             // VC 2008
 181             unsigned int *  pnDest = (unsigned int *)( ((unsigned __int64) pDest) & ~(unsigned __int64(3)) );
 182             unsigned int    nOffset = ((unsigned __int64) pDest) & 3;
 183             unsigned int    nExpected;
 184             unsigned int    nDesired;
 185
 186             for (;;) {
 187                 nExpected =
 188                     nDesired = *pnDest;
 189                 memcpy( reinterpret_cast<T *>(&nExpected) + nOffset, &expected, sizeof(T));
 190                 memcpy( reinterpret_cast<T *>(&nDesired) + nOffset, &desired, sizeof(T));
 191
 192                 unsigned int nPrev = (unsigned int) _InterlockedCompareExchange( (long *) pnDest, (long) nDesired, (long) nExpected );
 193                 if ( nPrev == nExpected )
 194                     return true;
 195                 T nByte;
 196                 memcpy( &nByte, reinterpret_cast<T *>(&nPrev) + nOffset, sizeof(T));
 197                 if ( nByte != expected ) {
 198                     expected = nByte;
 199                     return false;
 200                 }
 201             }
 202 #       endif
 203         }
 204 #if _MSC_VER >= 1600
 205 #   pragma warning(pop)
 206 #endif
 207
 208         template <typename T>
 209         static inline bool cas8_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 210         {
 211             return cas8_strong( pDest, expected, desired, mo_success, mo_fail );
 212         }
 213
 214 #if _MSC_VER >= 1600
 215 #   pragma warning(push)
 216         // Disable warning C4800: 'char' : forcing value to bool 'true' or 'false' (performance warning)
 217 #   pragma warning( disable: 4800 )
 218 #endif
 219         template <typename T>
 220         static inline T exchange8( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
 221         {
 222             static_assert( sizeof(T) == 1, "Illegal size of operand" );
 223
 224 #       if _MSC_VER >= 1600
 225             CDS_UNUSED(order);
 226             return (T) _InterlockedExchange8( (char volatile *) pDest, (char) v );
 227 #       else
 228             T expected = *pDest;
 229             do {} while ( !cas8_strong( pDest, expected, v, order, memory_order_relaxed ));
 230             return expected;
 231 #       endif
 232         }
 233 #if _MSC_VER >= 1600
 234 #   pragma warning(pop)
 235 #endif
 236
 237         template <typename T>
 238         static inline void store8( T volatile * pDest, T src, memory_order order ) CDS_NOEXCEPT
 239         {
 240             static_assert( sizeof(T) == 1, "Illegal size of operand" );
 241             assert( order ==  memory_order_relaxed
 242                 || order ==  memory_order_release
 243                 || order == memory_order_seq_cst
 244                 );
 245             assert( pDest != NULL );
 246
 247             if ( order != memory_order_seq_cst ) {
 248                 fence_before( order );
 249                 *pDest = src;
 250             }
 251             else {
 252                 exchange8( pDest, src, order );
 253             }
 254         }
 255
 256         template <typename T>
 257         static inline T load8( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
 258         {
 259             static_assert( sizeof(T) == 1, "Illegal size of operand" );
 260             assert( order ==  memory_order_relaxed
 261                 || order ==  memory_order_consume
 262                 || order ==  memory_order_acquire
 263                 || order == memory_order_seq_cst
 264                 );
 265             assert( pSrc != NULL );
 266
 267             T v = *pSrc;
 268             fence_after_load( order );
 269             return v;
 270         }
 271
 272         //-----------------------------------------------------------------------------
 273         // 16bit primitives
 274         //-----------------------------------------------------------------------------
 275
 276         template <typename T>
 277         static inline bool cas16_strong( T volatile * pDest, T& expected, T desired, memory_order /*mo_success*/, memory_order /*mo_fail*/ ) CDS_NOEXCEPT
 278         {
 279             static_assert( sizeof(T) == 2, "Illegal size of operand" );
 280             assert( cds::details::is_aligned( pDest, 2 ));
 281
 282             // _InterlockedCompareExchange behave as read-write memory barriers
 283             T prev = expected;
 284             expected = (T) _InterlockedCompareExchange16( (short *) pDest, (short) desired, (short) expected );
 285             return expected == prev;
 286         }
 287
 288         template <typename T>
 289         static inline bool cas16_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 290         {
 291             return cas16_strong( pDest, expected, desired, mo_success, mo_fail );
 292         }
 293
 294         template <typename T>
 295         static inline T exchange16( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
 296         {
 297             static_assert( sizeof(T) == 2, "Illegal size of operand" );
 298             assert( cds::details::is_aligned( pDest, 2 ));
 299
 300 #       if _MSC_VER >= 1600
 301             order;
 302             return (T) _InterlockedExchange16( (short volatile *) pDest, (short) v );
 303 #       else
 304             T expected = *pDest;
 305             do {} while ( !cas16_strong( pDest, expected, v, order, memory_order_relaxed ));
 306             return expected;
 307 #       endif
 308         }
 309
 310         template <typename T>
 311         static inline void store16( T volatile * pDest, T src, memory_order order ) CDS_NOEXCEPT
 312         {
 313             static_assert( sizeof(T) == 2, "Illegal size of operand" );
 314             assert( order ==  memory_order_relaxed
 315                 || order ==  memory_order_release
 316                 || order == memory_order_seq_cst
 317                 );
 318             assert( pDest != NULL );
 319             assert( cds::details::is_aligned( pDest, 2 ));
 320
 321             if ( order != memory_order_seq_cst ) {
 322                 fence_before( order );
 323                 *pDest = src;
 324             }
 325             else {
 326                 exchange16( pDest, src, order );
 327             }
 328         }
 329
 330         template <typename T>
 331         static inline T load16( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
 332         {
 333             static_assert( sizeof(T) == 2, "Illegal size of operand" );
 334             assert( order ==  memory_order_relaxed
 335                 || order ==  memory_order_consume
 336                 || order ==  memory_order_acquire
 337                 || order ==  memory_order_seq_cst
 338                 );
 339             assert( pSrc != NULL );
 340             assert( cds::details::is_aligned( pSrc, 2 ));
 341
 342             T v = *pSrc;
 343             fence_after_load( order );
 344             return v;
 345         }
 346
 347         //-----------------------------------------------------------------------------
 348         // 32bit primitives
 349         //-----------------------------------------------------------------------------
 350
 351         template <typename T>
 352         static inline T exchange32( T volatile * pDest, T v, memory_order /*order*/ ) CDS_NOEXCEPT
 353         {
 354             static_assert( sizeof(T) == 4, "Illegal size of operand" );
 355             assert( cds::details::is_aligned( pDest, 4 ));
 356
 357             return (T) _InterlockedExchange( (long *) pDest, (long) v );
 358         }
 359
 360         template <typename T>
 361         static inline void store32( T volatile * pDest, T src, memory_order order ) CDS_NOEXCEPT
 362         {
 363             static_assert( sizeof(T) == 4, "Illegal size of operand" );
 364             assert( order ==  memory_order_relaxed
 365                 || order ==  memory_order_release
 366                 || order == memory_order_seq_cst
 367                 );
 368             assert( pDest != NULL );
 369             assert( cds::details::is_aligned( pDest, 4 ));
 370
 371             if ( order != memory_order_seq_cst ) {
 372                 fence_before( order );
 373                 *pDest = src;
 374             }
 375             else {
 376                 exchange32( pDest, src, order );
 377             }
 378         }
 379
 380         template <typename T>
 381         static inline T load32( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
 382         {
 383             static_assert( sizeof(T) == 4, "Illegal size of operand" );
 384             assert( order ==  memory_order_relaxed
 385                 || order ==  memory_order_consume
 386                 || order ==  memory_order_acquire
 387                 || order ==  memory_order_seq_cst
 388                 );
 389             assert( pSrc != NULL );
 390             assert( cds::details::is_aligned( pSrc, 4 ));
 391
 392             T v = *pSrc;
 393             fence_after_load( order );
 394             return v;
 395         }
 396
 397         template <typename T>
 398         static inline bool cas32_strong( T volatile * pDest, T& expected, T desired, memory_order /*mo_success*/, memory_order /*mo_fail*/ ) CDS_NOEXCEPT
 399         {
 400             static_assert( sizeof(T) == 4, "Illegal size of operand" );
 401             assert( cds::details::is_aligned( pDest, 4 ));
 402
 403             // _InterlockedCompareExchange behave as read-write memory barriers
 404             T prev = expected;
 405             expected = (T) _InterlockedCompareExchange( (long *) pDest, (long) desired, (long) expected );
 406             return expected == prev;
 407         }
 408
 409         template <typename T>
 410         static inline bool cas32_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 411         {
 412             return cas32_strong( pDest, expected, desired, mo_success, mo_fail );
 413         }
 414
 415         // fetch_xxx may be emulated via cas32
 416         // If the platform has special fetch_xxx instruction
 417         // then it should define CDS_ATOMIC_fetch32_xxx_defined macro
 418
 419 #       define CDS_ATOMIC_fetch32_add_defined
 420         template <typename T>
 421         static inline T fetch32_add( T volatile * pDest, T v, memory_order /*order*/) CDS_NOEXCEPT
 422         {
 423             static_assert( sizeof(T) == 4, "Illegal size of operand" );
 424             assert( cds::details::is_aligned( pDest, 4 ));
 425
 426             // _InterlockedExchangeAdd behave as read-write memory barriers
 427             return (T) _InterlockedExchangeAdd( (long *) pDest, (long) v );
 428         }
 429
 430         //-----------------------------------------------------------------------------
 431         // 64bit primitives
 432         //-----------------------------------------------------------------------------
 433
 434         template <typename T>
 435         static inline bool cas64_strong( T volatile * pDest, T& expected, T desired, memory_order /*mo_success*/, memory_order /*mo_fail*/ ) CDS_NOEXCEPT
 436         {
 437             static_assert( sizeof(T) == 8, "Illegal size of operand" );
 438             assert( cds::details::is_aligned( pDest, 8 ));
 439
 440             // _InterlockedCompareExchange behave as read-write memory barriers
 441             T prev = expected;
 442             expected = (T) _InterlockedCompareExchange64( (__int64 *) pDest, (__int64) desired, (__int64) expected );
 443             return expected == prev;
 444         }
 445
 446         template <typename T>
 447         static inline bool cas64_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 448         {
 449             return cas64_strong( pDest, expected, desired, mo_success, mo_fail );
 450         }
 451
 452         template <typename T>
 453         static inline T load64( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
 454         {
 455             static_assert( sizeof(T) == 8, "Illegal size of operand" );
 456             assert( order ==  memory_order_relaxed
 457                 || order ==  memory_order_consume
 458                 || order ==  memory_order_acquire
 459                 || order ==  memory_order_seq_cst
 460                 );
 461             assert( pSrc != NULL );
 462             assert( cds::details::is_aligned( pSrc, 8 ));
 463
 464             T v = *pSrc;
 465             fence_after_load( order );
 466             return v;
 467         }
 468
 469
 470         template <typename T>
 471         static inline T exchange64( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
 472         {
 473             static_assert( sizeof(T) == 8, "Illegal size of operand" );
 474
 475             T cur = load64( pDest, memory_order_relaxed );
 476             do {
 477             } while (!cas64_weak( pDest, cur, v, order, memory_order_relaxed ));
 478             return cur;
 479         }
 480
 481         template <typename T>
 482         static inline void store64( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
 483         {
 484             static_assert( sizeof(T) == 8, "Illegal size of operand" );
 485             assert( order ==  memory_order_relaxed
 486                 || order ==  memory_order_release
 487                 || order == memory_order_seq_cst
 488                 );
 489             assert( pDest != NULL );
 490             assert( cds::details::is_aligned( pDest, 8 ));
 491
 492             if ( order != memory_order_seq_cst ) {
 493                 fence_before( order );
 494                 *pDest = val;
 495             }
 496             else {
 497                 exchange64( pDest, val, order );
 498             }
 499         }
 500
 501 #       define CDS_ATOMIC_fetch64_add_defined
 502         template <typename T>
 503         static inline T fetch64_add( T volatile * pDest, T v, memory_order /*order*/) CDS_NOEXCEPT
 504         {
 505             static_assert( sizeof(T) == 8, "Illegal size of operand" );
 506             assert( cds::details::is_aligned( pDest, 8 ));
 507
 508             // _InterlockedExchangeAdd64 behave as read-write memory barriers
 509             return (T) _InterlockedExchangeAdd64( (__int64 *) pDest, (__int64) v );
 510         }
 511
 512         //-----------------------------------------------------------------------------
 513         // pointer primitives
 514         //-----------------------------------------------------------------------------
 515
 516         template <typename T>
 517         static inline T * exchange_ptr( T * volatile * pDest, T * v, memory_order /*order*/ ) CDS_NOEXCEPT
 518         {
 519             static_assert( sizeof(T *) == sizeof(void *), "Illegal size of operand" );
 520             return (T *) _InterlockedExchangePointer( (void * volatile *) pDest, reinterpret_cast<void *>(v) );
 521         }
 522
 523         template <typename T>
 524         static inline void store_ptr( T * volatile * pDest, T * src, memory_order order ) CDS_NOEXCEPT
 525         {
 526             static_assert( sizeof(T *) == sizeof(void *), "Illegal size of operand" );
 527             assert( order ==  memory_order_relaxed
 528                 || order ==  memory_order_release
 529                 || order == memory_order_seq_cst
 530                 );
 531             assert( pDest != NULL );
 532
 533             if ( order != memory_order_seq_cst ) {
 534                 fence_before( order );
 535                 *pDest = src;
 536             }
 537             else {
 538                 exchange_ptr( pDest, src, order );
 539             }
 540         }
 541
 542         template <typename T>
 543         static inline T * load_ptr( T * volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
 544         {
 545             static_assert( sizeof(T *) == sizeof(void *), "Illegal size of operand" );
 546             assert( order ==  memory_order_relaxed
 547                 || order ==  memory_order_consume
 548                 || order ==  memory_order_acquire
 549                 || order ==  memory_order_seq_cst
 550                 );
 551             assert( pSrc != NULL );
 552
 553             T * v = *pSrc;
 554             fence_after_load( order );
 555             return v;
 556         }
 557
 558         template <typename T>
 559         static inline bool cas_ptr_strong( T * volatile * pDest, T *& expected, T * desired, memory_order /*mo_success*/, memory_order /*mo_fail*/ ) CDS_NOEXCEPT
 560         {
 561             static_assert( sizeof(T *) == sizeof(void *), "Illegal size of operand" );
 562
 563             // _InterlockedCompareExchangePointer behave as read-write memory barriers
 564             T * prev = expected;
 565             expected = (T *) _InterlockedCompareExchangePointer( (void * volatile *) pDest, (void *) desired, (void *) expected );
 566             return expected == prev;
 567         }
 568
 569         template <typename T>
 570         static inline bool cas_ptr_weak( T * volatile * pDest, T *& expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 571         {
 572             return cas_ptr_strong( pDest, expected, desired, mo_success, mo_fail );
 573         }
 574
 575     }} // namespace vc::amd64
 576
 577 #ifndef CDS_CXX11_INLINE_NAMESPACE_SUPPORT
 578     using namespace vc::amd64;
 579 #endif
 580     } // namespace platform
 581 }}  // namespace cds::cxx11_atomics
 582 //@endcond
 583
 584 #endif // #ifndef __CDS_COMPILER_VC_AMD64_CXX11_ATOMIC_H