cds/compiler/gcc/ia64/cxx11_atomic.h

   1 //$$CDS-header$$
   2
   3 #ifndef CDSLIB_COMPILER_GCC_IA64_CXX11_ATOMIC_H
   4 #define CDSLIB_COMPILER_GCC_IA64_CXX11_ATOMIC_H
   5
   6 /*
   7     Source:
   8         1. load/store: http://www.decadent.org.uk/pipermail/cpp-threads/2008-December/001932.html
   9         2. Mapping to C++ Memory Model: http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
  10 */
  11
  12 #include <cstdint>
  13
  14 //@cond
  15 namespace cds { namespace cxx11_atomic {
  16     namespace platform { CDS_CXX11_INLINE_NAMESPACE namespace gcc { CDS_CXX11_INLINE_NAMESPACE namespace ia64 {
  17
  18         static inline void itanium_full_fence() CDS_NOEXCEPT
  19         {
  20             __asm__ __volatile__ ( "mf \n\t" ::: "memory" );
  21         }
  22
  23         static inline void fence_before( memory_order order ) CDS_NOEXCEPT
  24         {
  25             switch(order) {
  26             case memory_order_relaxed:
  27             case memory_order_consume:
  28             case memory_order_acquire:
  29                 break;
  30             case memory_order_release:
  31             case memory_order_acq_rel:
  32                 CDS_COMPILER_RW_BARRIER;
  33                 break;
  34             case memory_order_seq_cst:
  35                 itanium_full_fence();
  36                 break;
  37             }
  38         }
  39
  40         static inline void fence_after( memory_order order ) CDS_NOEXCEPT
  41         {
  42             switch(order) {
  43             case memory_order_acquire:
  44             case memory_order_acq_rel:
  45                 CDS_COMPILER_RW_BARRIER;
  46                 break;
  47             case memory_order_relaxed:
  48             case memory_order_consume:
  49             case memory_order_release:
  50                 break;
  51             case memory_order_seq_cst:
  52                 itanium_full_fence();
  53                 break;
  54             }
  55         }
  56
  57
  58         //-----------------------------------------------------------------------------
  59         // fences
  60         //-----------------------------------------------------------------------------
  61         static inline void thread_fence(memory_order order) CDS_NOEXCEPT
  62         {
  63             switch(order)
  64             {
  65                 case memory_order_relaxed:
  66                 case memory_order_consume:
  67                     break;
  68                 case memory_order_release:
  69                 case memory_order_acquire:
  70                 case memory_order_acq_rel:
  71                     CDS_COMPILER_RW_BARRIER;
  72                     break;
  73                 case memory_order_seq_cst:
  74                     itanium_full_fence();
  75                     break;
  76                 default:;
  77             }
  78         }
  79
  80         static inline void signal_fence(memory_order order) CDS_NOEXCEPT
  81         {
  82             // C++11: 29.8.8: only compiler optimization, no hardware instructions
  83             switch(order)
  84             {
  85                 case memory_order_relaxed:
  86                     break;
  87                 case memory_order_consume:
  88                 case memory_order_release:
  89                 case memory_order_acquire:
  90                 case memory_order_acq_rel:
  91                 case memory_order_seq_cst:
  92                     CDS_COMPILER_RW_BARRIER;
  93                     break;
  94                 default:;
  95             }
  96         }
  97
  98 #define CDS_ITANIUM_ATOMIC_LOAD( n_bytes, n_bits )   \
  99         template <typename T>   \
 100         static inline T load##n_bits( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT \
 101         { \
 102             static_assert( sizeof(T) == n_bytes, "Illegal size of operand" )   ; \
 103             assert( order ==  memory_order_relaxed \
 104                 || order ==  memory_order_consume  \
 105                 || order ==  memory_order_acquire  \
 106                 || order == memory_order_seq_cst   \
 107                 ) ; \
 108             assert( pSrc )  ; \
 109             T val    ; \
 110             __asm__ __volatile__ ( \
 111                 "ld" #n_bytes ".acq %[val] = [%[pSrc]]  \n\t" \
 112                 : [val] "=r" (val) \
 113                 : [pSrc] "r" (pSrc) \
 114                 : "memory" \
 115                 ) ; \
 116             return val ; \
 117         }
 118
 119 #define CDS_ITANIUM_ATOMIC_STORE( n_bytes, n_bits ) \
 120         template <typename T> \
 121         static inline void store##n_bits( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT \
 122         { \
 123             static_assert( sizeof(T) == n_bytes, "Illegal size of operand" )   ; \
 124             assert( order ==  memory_order_relaxed \
 125                 || order ==  memory_order_release  \
 126                 || order == memory_order_seq_cst   \
 127                 ) ; \
 128             assert( pDest )  ; \
 129             if ( order == memory_order_seq_cst ) { \
 130                 __asm__ __volatile__ ( \
 131                     "st" #n_bytes ".rel [%[pDest]] = %[val] \n\t" \
 132                     "mf     \n\t" \
 133                     :: [pDest] "r" (pDest), [val] "r" (val) \
 134                     : "memory" \
 135                     ) ; \
 136             } \
 137             else { \
 138                 __asm__ __volatile__ ( \
 139                     "st" #n_bytes ".rel [%[pDest]] = %[val] \n\t" \
 140                     :: [pDest] "r" (pDest), [val] "r" (val) \
 141                     : "memory" \
 142                     ) ; \
 143                 fence_after(order) ; \
 144             } \
 145         }
 146
 147 #define CDS_ITANIUM_ATOMIC_CAS( n_bytes, n_bits ) \
 148         template <typename T> \
 149         static inline bool cas##n_bits##_strong( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order /*mo_fail*/ ) CDS_NOEXCEPT \
 150         { \
 151             static_assert( sizeof(T) == n_bytes, "Illegal size of operand" )   ; \
 152             T current ; \
 153             switch(mo_success) { \
 154             case memory_order_relaxed: \
 155             case memory_order_consume: \
 156             case memory_order_acquire: \
 157                 __asm__ __volatile__ ( \
 158                     "mov ar.ccv = %[expected] ;;\n\t" \
 159                     "cmpxchg" #n_bytes ".acq %[current] = [%[pDest]], %[desired], ar.ccv\n\t" \
 160                     : [current] "=r" (current) \
 161                     : [pDest] "r" (pDest), [expected] "r" (expected), [desired] "r" (desired) \
 162                     : "ar.ccv", "memory" \
 163                     ); \
 164                 break ; \
 165             case memory_order_release: \
 166                 __asm__ __volatile__ ( \
 167                     "mov ar.ccv = %[expected] ;;\n\t" \
 168                     "cmpxchg" #n_bytes ".rel %[current] = [%[pDest]], %[desired], ar.ccv\n\t" \
 169                     : [current] "=r" (current) \
 170                     : [pDest] "r" (pDest), [expected] "r" (expected), [desired] "r" (desired) \
 171                     : "ar.ccv", "memory" \
 172                     ); \
 173                 break ; \
 174             case memory_order_acq_rel: \
 175             case memory_order_seq_cst: \
 176                 __asm__ __volatile__ ( \
 177                     "mov ar.ccv = %[expected] ;;\n\t" \
 178                     "cmpxchg" #n_bytes ".rel %[current] = [%[pDest]], %[desired], ar.ccv\n\t" \
 179                     "mf \n\t" \
 180                     : [current] "=r" (current) \
 181                     : [pDest] "r" (pDest), [expected] "r" (expected), [desired] "r" (desired) \
 182                     : "ar.ccv", "memory" \
 183                     ); \
 184                 break; \
 185             default: \
 186                 assert(false); \
 187             } \
 188             bool bSuccess = expected == current ; \
 189             expected = current ; \
 190             return bSuccess ; \
 191         } \
 192         template <typename T> \
 193         static inline bool cas##n_bits##_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT \
 194         { return cas##n_bits##_strong( pDest, expected, desired, mo_success, mo_fail ); }
 195
 196         // xchg is performed with acquire semantics
 197 #define CDS_ITANIUM_ATOMIC_EXCHANGE( n_bytes, n_bits ) \
 198         template <typename T> \
 199         static inline T exchange##n_bits( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT \
 200         { \
 201             static_assert( sizeof(T) == n_bytes, "Illegal size of operand" )   ; \
 202             assert( pDest ) ; \
 203             T current ; \
 204             switch(order) \
 205             { \
 206             case memory_order_relaxed: \
 207             case memory_order_consume: \
 208             case memory_order_acquire: \
 209                 __asm__ __volatile__ ( \
 210                     "xchg" #n_bytes " %[current] = [%[pDest]], %[val]\n\t" \
 211                     : [current] "=r" (current) \
 212                     : [pDest] "r" (pDest), [val] "r" (val) \
 213                     : "memory" \
 214                     ); \
 215                 break; \
 216             case memory_order_acq_rel: \
 217             case memory_order_release: \
 218             case memory_order_seq_cst: \
 219                 __asm__ __volatile__ ( \
 220                     "mf \n\t" \
 221                     "xchg" #n_bytes " %[current] = [%[pDest]], %[val]\n\t" \
 222                     : [current] "=r" (current) \
 223                     : [pDest] "r" (pDest), [val] "r" (val) \
 224                     : "memory" \
 225                     ); \
 226                 break; \
 227             default: assert(false); \
 228             } \
 229             return current ; \
 230         }
 231
 232 #define CDS_ITANIUM_ATOMIC_FETCH_ADD( n_bytes, n_add )  \
 233         switch (order) { \
 234             case memory_order_relaxed: \
 235             case memory_order_consume: \
 236             case memory_order_acquire: \
 237                 __asm__ __volatile__ ( \
 238                     "fetchadd" #n_bytes ".acq %[cur] = [%[pDest]], " #n_add " \n\t" \
 239                     : [cur] "=r" (cur) \
 240                     : [pDest] "r" (pDest) \
 241                     : "memory" \
 242                     ); \
 243                 break ; \
 244             case memory_order_release: \
 245                 __asm__ __volatile__ ( \
 246                     "fetchadd" #n_bytes ".rel %[cur] = [%[pDest]], " #n_add " \n\t" \
 247                     : [cur] "=r" (cur) \
 248                     : [pDest] "r" (pDest) \
 249                     : "memory" \
 250                     ); \
 251                 break ; \
 252             case memory_order_acq_rel: \
 253             case memory_order_seq_cst: \
 254                 __asm__ __volatile__ ( \
 255                     "fetchadd" #n_bytes ".rel %[cur] = [%[pDest]], " #n_add " \n\t" \
 256                     "mf \n\t" \
 257                     : [cur] "=r" (cur) \
 258                     : [pDest] "r" (pDest) \
 259                     : "memory" \
 260                     ); \
 261                 break ; \
 262             default: \
 263                 assert(false); \
 264         }
 265
 266         //-----------------------------------------------------------------------------
 267         // 8bit primitives
 268         //-----------------------------------------------------------------------------
 269
 270         CDS_ITANIUM_ATOMIC_LOAD( 1, 8 )
 271         CDS_ITANIUM_ATOMIC_STORE( 1, 8 )
 272         CDS_ITANIUM_ATOMIC_CAS( 1, 8 )
 273         CDS_ITANIUM_ATOMIC_EXCHANGE( 1, 8 )
 274
 275         //-----------------------------------------------------------------------------
 276         // 16bit primitives
 277         //-----------------------------------------------------------------------------
 278
 279         CDS_ITANIUM_ATOMIC_LOAD( 2, 16 )
 280         CDS_ITANIUM_ATOMIC_STORE( 2, 16 )
 281         CDS_ITANIUM_ATOMIC_CAS( 2, 16 )
 282         CDS_ITANIUM_ATOMIC_EXCHANGE( 2, 16 )
 283
 284         //-----------------------------------------------------------------------------
 285         // 32bit primitives
 286         //-----------------------------------------------------------------------------
 287
 288         CDS_ITANIUM_ATOMIC_LOAD( 4, 32 )
 289         CDS_ITANIUM_ATOMIC_STORE( 4, 32 )
 290         CDS_ITANIUM_ATOMIC_CAS( 4, 32 )
 291         CDS_ITANIUM_ATOMIC_EXCHANGE( 4, 32 )
 292
 293 #       define CDS_ATOMIC_fetch32_add_defined
 294         template <typename T>
 295         static inline T fetch32_add( T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 296         {
 297             static_assert( sizeof(T) == 4, "Illegal size of operand" );
 298             assert( pDest );
 299
 300             T cur;
 301             switch ( val ) {
 302                 case 1:
 303                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, 1 );
 304                     break;
 305                 case 4:
 306                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, 4 );
 307                     break;
 308                 case 8:
 309                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, 8 );
 310                     break;
 311                 case 16:
 312                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, 16 );
 313                     break;
 314                 default:
 315                     cur = load32( pDest, memory_order_relaxed );
 316                     do {} while ( !cas32_strong( pDest, cur, cur + val, order, memory_order_relaxed ));
 317                     break;
 318             }
 319             return cur;
 320         }
 321
 322 #       define CDS_ATOMIC_fetch32_sub_defined
 323         template <typename T>
 324         static inline T fetch32_sub( T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 325         {
 326             static_assert( sizeof(T) == 4, "Illegal size of operand" );
 327             assert( pDest );
 328             T cur;
 329             switch ( val ) {
 330                 case 1:
 331                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, -1 );
 332                     break;
 333                 case 4:
 334                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, -4 );
 335                     break;
 336                 case 8:
 337                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, -8 );
 338                     break;
 339                 case 16:
 340                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, -16 );
 341                     break;
 342                 default:
 343                     cur = load32( pDest, memory_order_relaxed );
 344                     do {} while ( !cas32_strong( pDest, cur, cur - val, order, memory_order_relaxed ));
 345                     break;
 346             }
 347             return cur;
 348         }
 349
 350         //-----------------------------------------------------------------------------
 351         // 64bit primitives
 352         //-----------------------------------------------------------------------------
 353
 354         CDS_ITANIUM_ATOMIC_LOAD( 8, 64 )
 355         CDS_ITANIUM_ATOMIC_STORE( 8, 64 )
 356         CDS_ITANIUM_ATOMIC_CAS( 8, 64 )
 357         CDS_ITANIUM_ATOMIC_EXCHANGE( 8, 64 )
 358
 359 #       define CDS_ATOMIC_fetch64_add_defined
 360         template <typename T>
 361         static inline T fetch64_add( T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 362         {
 363             static_assert( sizeof(T) == 8, "Illegal size of operand" );
 364             assert( pDest );
 365
 366             T cur;
 367             switch ( val ) {
 368                 case 1:
 369                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 1 );
 370                     break;
 371                 case 4:
 372                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 4 );
 373                     break;
 374                 case 8:
 375                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 8 );
 376                     break;
 377                 case 16:
 378                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 16 );
 379                     break;
 380                 default:
 381                     cur = load64( pDest, memory_order_relaxed );
 382                     do {} while ( !cas64_strong( pDest, cur, cur + val, order, memory_order_relaxed ));
 383                     break;
 384             }
 385             return cur;
 386         }
 387
 388 #       define CDS_ATOMIC_fetch64_sub_defined
 389         template <typename T>
 390         static inline T fetch64_sub( T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
 391         {
 392             static_assert( sizeof(T) == 8, "Illegal size of operand" );
 393             assert( pDest );
 394             T cur;
 395             switch ( val ) {
 396                 case 1:
 397                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -1 );
 398                     break;
 399                 case 4:
 400                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -4 );
 401                     break;
 402                 case 8:
 403                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -8 );
 404                     break;
 405                 case 16:
 406                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -16 );
 407                     break;
 408                 default:
 409                     cur = load64( pDest, memory_order_relaxed );
 410                     do {} while ( !cas64_strong( pDest, cur, cur - val, order, memory_order_relaxed ));
 411                     break;
 412             }
 413             return cur;
 414         }
 415
 416         //-----------------------------------------------------------------------------
 417         // pointer primitives
 418         //-----------------------------------------------------------------------------
 419         template <typename T>
 420         static inline T * load_ptr( T * volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
 421         {
 422             assert( order ==  memory_order_relaxed
 423                  || order ==  memory_order_consume
 424                  || order ==  memory_order_acquire
 425                  || order == memory_order_seq_cst
 426             );
 427             assert( pSrc );
 428             T * val;
 429             __asm__ __volatile__ (
 430                 "ld8.acq %[val] = [%[pSrc]]  \n\t"
 431                 : [val] "=r" (val)
 432                 : [pSrc] "r" (pSrc)
 433                 : "memory"
 434             );
 435             return val;
 436         }
 437
 438         template <typename T>
 439         static inline void store_ptr( T * volatile * pDest, T * val, memory_order order ) CDS_NOEXCEPT
 440         {
 441             assert( order ==  memory_order_relaxed
 442                  || order ==  memory_order_release
 443                  || order == memory_order_seq_cst
 444             );
 445             assert( pDest );
 446
 447             if ( order == memory_order_seq_cst ) {
 448                 __asm__ __volatile__ (
 449                     "st8.rel [%[pDest]] = %[val] \n\t"
 450                     "mf     \n\t"
 451                     :: [pDest] "r" (pDest), [val] "r" (val)
 452                     : "memory"
 453                 );
 454             }
 455             else {
 456                 __asm__ __volatile__ (
 457                     "st8.rel [%[pDest]] = %[val] \n\t"
 458                     :: [pDest] "r" (pDest), [val] "r" (val)
 459                     : "memory"
 460                 );
 461                 fence_after(order);
 462             }
 463         }
 464
 465         template <typename T>
 466         static inline bool cas_ptr_strong( T * volatile * pDest, T *& expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 467         {
 468             static_assert( sizeof(T *) == 8, "Illegal size of operand" );
 469             assert( pDest );
 470
 471             T * current;
 472
 473             switch(mo_success) {
 474             case memory_order_relaxed:
 475             case memory_order_consume:
 476             case memory_order_acquire:
 477                 __asm__ __volatile__ (
 478                     "mov ar.ccv = %[expected] ;;\n\t"
 479                     "cmpxchg8.acq %[current] = [%[pDest]], %[desired], ar.ccv\n\t"
 480                     : [current] "=r" (current)
 481                     : [pDest] "r" (pDest), [expected] "r" (expected), [desired] "r" (desired)
 482                     : "ar.ccv", "memory"
 483                 );
 484                 break;
 485             case memory_order_release:
 486                __asm__ __volatile__ (
 487                    "mov ar.ccv = %[expected] ;;\n\t"
 488                    "cmpxchg8.rel %[current] = [%[pDest]], %[desired], ar.ccv\n\t"
 489                    : [current] "=r" (current)
 490                    : [pDest] "r" (pDest), [expected] "r" (expected), [desired] "r" (desired)
 491                    : "ar.ccv", "memory"
 492                );
 493                break;
 494             case memory_order_acq_rel:
 495             case memory_order_seq_cst:
 496                __asm__ __volatile__ (
 497                    "mov ar.ccv = %[expected] ;;\n\t"
 498                    "cmpxchg8.rel %[current] = [%[pDest]], %[desired], ar.ccv\n\t"
 499                    "mf \n\t"
 500                    : [current] "=r" (current)
 501                    : [pDest] "r" (pDest), [expected] "r" (expected), [desired] "r" (desired)
 502                    : "ar.ccv", "memory"
 503                );
 504                break;
 505             default:
 506                 assert(false);
 507             }
 508
 509             bool bSuccess = expected == current;
 510             expected = current;
 511             if ( !bSuccess )
 512                 fence_after( mo_fail );
 513             return bSuccess;
 514         }
 515
 516         template <typename T>
 517         static inline bool cas_ptr_weak( T * volatile * pDest, T *& expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
 518         {
 519             return cas_ptr_strong( pDest, expected, desired, mo_success, mo_fail );
 520         }
 521
 522         template <typename T>
 523         static inline T * exchange_ptr( T * volatile * pDest, T * val, memory_order order ) CDS_NOEXCEPT
 524         {
 525             static_assert( sizeof(T *) == 8, "Illegal size of operand" );
 526             assert( pDest );
 527
 528             T * current;
 529             switch(order) {
 530                 case memory_order_relaxed:
 531                 case memory_order_consume:
 532                 case memory_order_acquire:
 533                     __asm__ __volatile__ (
 534                         "xchg8 %[current] = [%[pDest]], %[val]\n\t"
 535                         : [current] "=r" (current)
 536                         : [pDest] "r" (pDest), [val] "r" (val)
 537                         : "memory"
 538                     );
 539                     break;
 540                 case memory_order_acq_rel:
 541                 case memory_order_release:
 542                 case memory_order_seq_cst:
 543                     __asm__ __volatile__ (
 544                         "mf \n\t"
 545                         "xchg8 %[current] = [%[pDest]], %[val]\n\t"
 546                         : [current] "=r" (current)
 547                         : [pDest] "r" (pDest), [val] "r" (val)
 548                         : "memory"
 549                     );
 550                     break;
 551                 default: assert(false);
 552             }
 553             return current;
 554         }
 555
 556
 557         template <typename T> struct atomic_pointer_sizeof { enum { value = sizeof(T) }; };
 558         template <> struct atomic_pointer_sizeof<void> { enum { value = 1 }; };
 559
 560         // It does not work properly
 561         // atomic.fetch_add( ... ) returns nullptr, why?..
 562 //#       define CDS_ATOMIC_fetch_ptr_add_defined
 563         template <typename T>
 564         static inline T * fetch_ptr_add( T * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
 565         {
 566             static_assert( sizeof(T *) == 8, "Illegal size of operand" );
 567             assert( pDest );
 568
 569             T * cur;
 570             val *= atomic_pointer_sizeof<T>::value;
 571             switch ( val ) {
 572                 case 1:
 573                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 1 );
 574                     break;
 575                 case 4:
 576                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 4 );
 577                     break;
 578                 case 8:
 579                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 8 );
 580                     break;
 581                 case 16:
 582                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 16 );
 583                     break;
 584                 default:
 585                     cur = load_ptr( pDest, memory_order_relaxed );
 586                     do {} while ( !cas_ptr_strong( pDest, cur, reinterpret_cast<T *>(reinterpret_cast<uint8_t *>(cur) + val), order, memory_order_relaxed ));
 587                     break;
 588             }
 589             return cur;
 590         }
 591
 592         // It does not work properly
 593         // atomic.fetch_sub( ... ) returns nullptr, why?..
 594 //#       define CDS_ATOMIC_fetch_ptr_sub_defined
 595         template <typename T>
 596         static inline T * fetch_ptr_sub( T * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
 597         {
 598             static_assert( sizeof(T *) == 8, "Illegal size of operand" );
 599             assert( pDest );
 600             T * cur;
 601             val *= atomic_pointer_sizeof<T>::value;
 602             switch ( val ) {
 603                 case 1:
 604                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -1 );
 605                     break;
 606                 case 4:
 607                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -4 );
 608                     break;
 609                 case 8:
 610                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -8 );
 611                     break;
 612                 case 16:
 613                     CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -16 );
 614                     break;
 615                 default:
 616                     cur = load_ptr( pDest, memory_order_relaxed );
 617                     do {} while ( !cas_ptr_strong( pDest, cur, reinterpret_cast<T *>(reinterpret_cast<uint8_t *>(cur) - val), order, memory_order_relaxed ));
 618                     break;
 619             }
 620             return cur;
 621         }
 622
 623         //-----------------------------------------------------------------------------
 624         // atomic flag primitives
 625         //-----------------------------------------------------------------------------
 626
 627         typedef bool atomic_flag_type;
 628         static inline bool atomic_flag_tas( atomic_flag_type volatile * pFlag, memory_order order ) CDS_NOEXCEPT
 629         {
 630             return exchange8( pFlag, true, order );
 631         }
 632
 633         static inline void atomic_flag_clear( atomic_flag_type volatile * pFlag, memory_order order ) CDS_NOEXCEPT
 634         {
 635             store8( pFlag, false, order );
 636         }
 637
 638 #undef CDS_ITANIUM_ATOMIC_LOAD
 639 #undef CDS_ITANIUM_ATOMIC_STORE
 640 #undef CDS_ITANIUM_ATOMIC_CAS
 641 #undef CDS_ITANIUM_ATOMIC_EXCHANGE
 642 #undef CDS_ITANIUM_ATOMIC_FETCH_ADD
 643
 644     }} // namespace gcc::ia64
 645
 646 #ifndef CDS_CXX11_INLINE_NAMESPACE_SUPPORT
 647     using namespace gcc::ia64;
 648 #endif
 649     }   // namespace platform
 650 }}  // namespace cds::cxx11_atomic
 651 //@endcond
 652
 653 #endif // #ifndef CDSLIB_COMPILER_GCC_IA64_CXX11_ATOMIC_H