cds/compiler/gcc/x86/cxx11_atomic32.h

   1 //$$CDS-header$$
   2
   3 #ifndef CDSLIB_COMPILER_GCC_X86_CXX11_ATOMIC32_H
   4 #define CDSLIB_COMPILER_GCC_X86_CXX11_ATOMIC32_H
   5
   6 #include <cstdint>
   7 #include <cds/details/is_aligned.h>
   8
   9 //@cond
  10 namespace cds { namespace cxx11_atomic {
  11     namespace platform { CDS_CXX11_INLINE_NAMESPACE namespace gcc { CDS_CXX11_INLINE_NAMESPACE namespace x86 {
  12
  13         static inline void fence_before( memory_order order ) CDS_NOEXCEPT
  14         {
  15             switch(order) {
  16             case memory_order_relaxed:
  17             case memory_order_acquire:
  18             case memory_order_consume:
  19                 break;
  20             case memory_order_release:
  21             case memory_order_acq_rel:
  22                 CDS_COMPILER_RW_BARRIER;
  23                 break;
  24             case memory_order_seq_cst:
  25                 CDS_COMPILER_RW_BARRIER;
  26                 break;
  27             }
  28         }
  29
  30         static inline void fence_after( memory_order order ) CDS_NOEXCEPT
  31         {
  32             switch(order) {
  33             case memory_order_acquire:
  34             case memory_order_acq_rel:
  35                 CDS_COMPILER_RW_BARRIER;
  36                 break;
  37             case memory_order_relaxed:
  38             case memory_order_consume:
  39             case memory_order_release:
  40                 break;
  41             case memory_order_seq_cst:
  42                 CDS_COMPILER_RW_BARRIER;
  43                 break;
  44             }
  45         }
  46
  47
  48         static inline void fence_after_load(memory_order order) CDS_NOEXCEPT
  49         {
  50             switch(order) {
  51             case memory_order_relaxed:
  52             case memory_order_release:
  53                 break;
  54             case memory_order_acquire:
  55             case memory_order_acq_rel:
  56                 CDS_COMPILER_RW_BARRIER;
  57                 break;
  58             case memory_order_consume:
  59                 break;
  60             case memory_order_seq_cst:
  61                 __asm__ __volatile__ ( "mfence" ::: "memory"  );
  62                 break;
  63             default:;
  64             }
  65         }
  66
  67         //-----------------------------------------------------------------------------
  68         // fences
  69         //-----------------------------------------------------------------------------
  70         static inline void thread_fence(memory_order order) CDS_NOEXCEPT
  71         {
  72             switch(order)
  73             {
  74                 case memory_order_relaxed:
  75                 case memory_order_consume:
  76                     break;
  77                 case memory_order_release:
  78                 case memory_order_acquire:
  79                 case memory_order_acq_rel:
  80                     CDS_COMPILER_RW_BARRIER;
  81                     break;
  82                 case memory_order_seq_cst:
  83                     __asm__ __volatile__ ( "mfence" ::: "memory"  );
  84                     break;
  85                 default:;
  86             }
  87         }
  88
  89         static inline void signal_fence(memory_order order) CDS_NOEXCEPT
  90         {
  91             // C++11: 29.8.8: only compiler optimization, no hardware instructions
  92             switch(order)
  93             {
  94                 case memory_order_relaxed:
  95                     break;
  96                 case memory_order_consume:
  97                 case memory_order_release:
  98                 case memory_order_acquire:
  99                 case memory_order_acq_rel:
 100                 case memory_order_seq_cst:
 101                     CDS_COMPILER_RW_BARRIER;
 102                     break;
 103                 default:;
 104             }
 105         }
 106
 107         //-----------------------------------------------------------------------------
 108         // 8bit primitives
 109         //-----------------------------------------------------------------------------
 110
 111         template <typename T>
 112         static inline bool cas8_strong( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 113         {
 114             static_assert( sizeof(T) == 1, "Illegal size of operand" );
 115
 116             T prev = expected;
 117             fence_before(mo_success);
 118             __asm__ __volatile__  (
 119                 "lock ; cmpxchgb %[desired], %[pDest]"
 120                 : [prev] "+a" (prev), [pDest] "+m" (*pDest)
 121                 : [desired] "q" (desired)
 122                 );
 123             bool success = (prev == expected);
 124             expected = prev;
 125             if (success)
 126                 fence_after(mo_success);
 127             else
 128                 fence_after(mo_fail);
 129             return success;
 130         }
 131
 132         template <typename T>
 133         static inline bool cas8_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 134         {
 135             return cas8_strong( pDest, expected, desired, mo_success, mo_fail );
 136         }
 137
 138         template <typename T>
 139         static inline T exchange8( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
 140         {
 141             static_assert( sizeof(T) == 1, "Illegal size of operand" );
 142
 143             fence_before(order);
 144             __asm__ __volatile__  (
 145                 "xchgb %[v], %[pDest]"
 146                 : [v] "+q" (v), [pDest] "+m" (*pDest)
 147                 );
 148             fence_after(order);
 149             return v;
 150         }
 151
 152         template <typename T>
 153         static inline void store8( T volatile * pDest, T src, memory_order order ) CDS_NOEXCEPT
 154         {
 155             static_assert( sizeof(T) == 1, "Illegal size of operand" );
 156             assert( order ==  memory_order_relaxed
 157                 || order ==  memory_order_release
 158                 || order == memory_order_seq_cst
 159                 );
 160             assert( pDest != NULL );
 161
 162             if ( order != memory_order_seq_cst ) {
 163                 fence_before( order );
 164                 *pDest = src;
 165             }
 166             else {
 167                 exchange8( pDest, src, order );
 168             }
 169         }
 170
 171         template <typename T>
 172         static inline T load8( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
 173         {
 174             static_assert( sizeof(T) == 1, "Illegal size of operand" );
 175             assert( order ==  memory_order_relaxed
 176                 || order ==  memory_order_consume
 177                 || order ==  memory_order_acquire
 178                 || order == memory_order_seq_cst
 179                 );
 180             assert( pSrc != NULL );
 181
 182             T v = *pSrc;
 183             fence_after_load( order );
 184             return v;
 185         }
 186
 187 #       define CDS_ATOMIC_fetch8_add_defined
 188         template <typename T>
 189         static inline T fetch8_add( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
 190         {
 191             fence_before(order);
 192             __asm__ __volatile__  (
 193                 "lock ; xaddb %[val], %[pDest]"
 194                 : [val] "+q" (val), [pDest] "+m" (*pDest)
 195                 );
 196             fence_after(order);
 197             return val;
 198         }
 199
 200 #       define CDS_ATOMIC_fetch8_sub_defined
 201         template <typename T>
 202         static inline T fetch8_sub( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
 203         {
 204             fence_before(order);
 205             __asm__ __volatile__  (
 206                 "negb %[val] ; \n"
 207                 "lock ; xaddb %[val], %[pDest]"
 208                 : [val] "+q" (val), [pDest] "+m" (*pDest)
 209                 );
 210             fence_after(order);
 211             return val;
 212         }
 213
 214         //-----------------------------------------------------------------------------
 215         // atomic flag primitives
 216         //-----------------------------------------------------------------------------
 217
 218         typedef bool atomic_flag_type;
 219         static inline bool atomic_flag_tas( atomic_flag_type volatile * pFlag, memory_order order ) CDS_NOEXCEPT
 220         {
 221             return exchange8( pFlag, true, order );
 222         }
 223
 224         static inline void atomic_flag_clear( atomic_flag_type volatile * pFlag, memory_order order ) CDS_NOEXCEPT
 225         {
 226             store8( pFlag, false, order );
 227         }
 228
 229         //-----------------------------------------------------------------------------
 230         // 16bit primitives
 231         //-----------------------------------------------------------------------------
 232
 233         template <typename T>
 234         static inline T exchange16( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
 235         {
 236             static_assert( sizeof(T) == 2, "Illegal size of operand" );
 237             assert( cds::details::is_aligned( pDest, 2 ));
 238
 239             fence_before(order);
 240             __asm__ __volatile__  (
 241                 "xchgw %[v], %[pDest]"
 242                 : [v] "+q" (v), [pDest] "+m" (*pDest)
 243                 );
 244             fence_after(order);
 245             return v;
 246         }
 247
 248         template <typename T>
 249         static inline void store16( T volatile * pDest, T src, memory_order order ) CDS_NOEXCEPT
 250         {
 251             static_assert( sizeof(T) == 2, "Illegal size of operand" );
 252             assert( order ==  memory_order_relaxed
 253                 || order ==  memory_order_release
 254                 || order == memory_order_seq_cst
 255                 );
 256             assert( pDest != NULL );
 257             assert( cds::details::is_aligned( pDest, 2 ));
 258
 259             if ( order != memory_order_seq_cst ) {
 260                 fence_before( order );
 261                 *pDest = src;
 262             }
 263             else {
 264                 exchange16( pDest, src, order );
 265             }
 266         }
 267
 268         template <typename T>
 269         static inline T load16( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
 270         {
 271             static_assert( sizeof(T) == 2, "Illegal size of operand" );
 272             assert( order ==  memory_order_relaxed
 273                 || order ==  memory_order_consume
 274                 || order ==  memory_order_acquire
 275                 || order ==  memory_order_seq_cst
 276                 );
 277             assert( pSrc != NULL );
 278             assert( cds::details::is_aligned( pSrc, 2 ));
 279
 280             T v = *pSrc;
 281             fence_after_load( order );
 282             return v;
 283         }
 284
 285         template <typename T>
 286         static inline bool cas16_strong( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 287         {
 288             static_assert( sizeof(T) == 2, "Illegal size of operand" );
 289             assert( cds::details::is_aligned( pDest, 2 ));
 290
 291             T prev = expected;
 292             fence_before(mo_success);
 293             __asm__ __volatile__  (
 294                 "lock ; cmpxchgw %[desired], %[pDest]"
 295                 : [prev] "+a" (prev), [pDest] "+m" (*pDest)
 296                 : [desired] "q" (desired)
 297                 );
 298             bool success = prev == expected;
 299             if (success)
 300                 fence_after(mo_success);
 301             else {
 302                 fence_after(mo_fail);
 303                 expected = prev;
 304             }
 305
 306             return success;
 307         }
 308
 309         template <typename T>
 310         static inline bool cas16_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 311         {
 312             return cas16_strong( pDest, expected, desired, mo_success, mo_fail );
 313         }
 314
 315 #       define CDS_ATOMIC_fetch16_add_defined
 316         template <typename T>
 317         static inline T fetch16_add( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
 318         {
 319             static_assert( sizeof(T) == 2, "Illegal size of operand" );
 320             assert( cds::details::is_aligned( pDest, 2 ));
 321
 322             fence_before(order);
 323             __asm__ __volatile__  (
 324                 "lock ; xaddw %[val], %[pDest]"
 325                 : [val] "+q" (val), [pDest] "+m" (*pDest)
 326                 );
 327             fence_after(order);
 328             return val;
 329         }
 330
 331 #       define CDS_ATOMIC_fetch16_sub_defined
 332         template <typename T>
 333         static inline T fetch16_sub( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
 334         {
 335             static_assert( sizeof(T) == 2, "Illegal size of operand" );
 336             assert( cds::details::is_aligned( pDest, 2 ));
 337
 338             fence_before(order);
 339             __asm__ __volatile__  (
 340                 "negw %[val] ; \n"
 341                 "lock ; xaddw %[val], %[pDest]"
 342                 : [val] "+q" (val), [pDest] "+m" (*pDest)
 343                 );
 344             fence_after(order);
 345             return val;
 346         }
 347
 348         //-----------------------------------------------------------------------------
 349         // 32bit primitives
 350         //-----------------------------------------------------------------------------
 351
 352         template <typename T>
 353         static inline T exchange32( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
 354         {
 355             static_assert( sizeof(T) == 4, "Illegal size of operand" );
 356             assert( cds::details::is_aligned( pDest, 4 ));
 357
 358             fence_before(order);
 359             __asm__ __volatile__  (
 360                 "xchgl %[v], %[pDest]"
 361                 : [v] "+r" (v), [pDest] "+m" (*pDest)
 362                 );
 363             fence_after(order);
 364             return v;
 365         }
 366
 367         template <typename T>
 368         static inline void store32( T volatile * pDest, T src, memory_order order ) CDS_NOEXCEPT
 369         {
 370             static_assert( sizeof(T) == 4, "Illegal size of operand" );
 371             assert( order ==  memory_order_relaxed
 372                 || order ==  memory_order_release
 373                 || order == memory_order_seq_cst
 374                 );
 375             assert( pDest != NULL );
 376             assert( cds::details::is_aligned( pDest, 4 ));
 377
 378             if ( order != memory_order_seq_cst ) {
 379                 fence_before( order );
 380                 *pDest = src;
 381             }
 382             else {
 383                 exchange32( pDest, src, order );
 384             }
 385         }
 386
 387         template <typename T>
 388         static inline T load32( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
 389         {
 390             static_assert( sizeof(T) == 4, "Illegal size of operand" );
 391             assert( order ==  memory_order_relaxed
 392                 || order ==  memory_order_consume
 393                 || order ==  memory_order_acquire
 394                 || order ==  memory_order_seq_cst
 395                 );
 396             assert( pSrc != NULL );
 397             assert( cds::details::is_aligned( pSrc, 4 ));
 398
 399             T v = *pSrc;
 400             fence_after_load( order );
 401             return v;
 402         }
 403
 404         template <typename T>
 405         static inline bool cas32_strong( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 406         {
 407             static_assert( sizeof(T) == 4, "Illegal size of operand" );
 408             assert( cds::details::is_aligned( pDest, 4 ));
 409
 410             T prev = expected;
 411             fence_before(mo_success);
 412             __asm__ __volatile__  (
 413                 "lock ; cmpxchgl %[desired], %[pDest]"
 414                 : [prev] "+a" (prev), [pDest] "+m" (*pDest)
 415                 : [desired] "r" (desired)
 416                 );
 417             bool success = prev == expected;
 418             if (success)
 419                 fence_after(mo_success);
 420             else {
 421                 fence_after(mo_fail);
 422                 expected = prev;
 423             }
 424             return success;
 425         }
 426
 427         template <typename T>
 428         static inline bool cas32_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 429         {
 430             return cas32_strong( pDest, expected, desired, mo_success, mo_fail );
 431         }
 432
 433         // fetch_xxx may be emulated via cas32
 434         // If the platform has special fetch_xxx instruction
 435         // then it should define CDS_ATOMIC_fetch32_xxx_defined macro
 436
 437 #       define CDS_ATOMIC_fetch32_add_defined
 438         template <typename T>
 439         static inline T fetch32_add( T volatile * pDest, T v, memory_order order) CDS_NOEXCEPT
 440         {
 441             static_assert( sizeof(T) == 4, "Illegal size of operand" );
 442             assert( cds::details::is_aligned( pDest, 4 ));
 443
 444             fence_before(order);
 445             __asm__ __volatile__  (
 446                 "lock ; xaddl %[v], %[pDest]"
 447                 : [v] "+r" (v), [pDest] "+m" (*pDest)
 448                 );
 449             fence_after(order);
 450             return v;
 451         }
 452
 453 #       define CDS_ATOMIC_fetch32_sub_defined
 454         template <typename T>
 455         static inline T fetch32_sub( T volatile * pDest, T v, memory_order order) CDS_NOEXCEPT
 456         {
 457             static_assert( sizeof(T) == 4, "Illegal size of operand" );
 458             assert( cds::details::is_aligned( pDest, 4 ));
 459
 460             fence_before(order);
 461             __asm__ __volatile__  (
 462                 "negl   %[v] ; \n"
 463                 "lock ; xaddl %[v], %[pDest]"
 464                 : [v] "+r" (v), [pDest] "+m" (*pDest)
 465                 );
 466             fence_after(order);
 467             return v;
 468         }
 469
 470     }}} // namespace platform::gcc::x86
 471 }}  // namespace cds::cxx11_atomic
 472 //@endcond
 473
 474 #endif // #ifndef CDSLIB_COMPILER_GCC_X86_CXX11_ATOMIC32_H