3 #ifndef __CDS_COMPILER_GCC_IA64_CXX11_ATOMIC_H
4 #define __CDS_COMPILER_GCC_IA64_CXX11_ATOMIC_H
8 1. load/store: http://www.decadent.org.uk/pipermail/cpp-threads/2008-December/001932.html
9 2. Mapping to C++ Memory Model: http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
15 namespace cds { namespace cxx11_atomic {
16 namespace platform { CDS_CXX11_INLINE_NAMESPACE namespace gcc { CDS_CXX11_INLINE_NAMESPACE namespace ia64 {
18 static inline void itanium_full_fence() CDS_NOEXCEPT
20 __asm__ __volatile__ ( "mf \n\t" ::: "memory" );
23 static inline void fence_before( memory_order order ) CDS_NOEXCEPT
26 case memory_order_relaxed:
27 case memory_order_consume:
28 case memory_order_acquire:
30 case memory_order_release:
31 case memory_order_acq_rel:
32 CDS_COMPILER_RW_BARRIER;
34 case memory_order_seq_cst:
40 static inline void fence_after( memory_order order ) CDS_NOEXCEPT
43 case memory_order_acquire:
44 case memory_order_acq_rel:
45 CDS_COMPILER_RW_BARRIER;
47 case memory_order_relaxed:
48 case memory_order_consume:
49 case memory_order_release:
51 case memory_order_seq_cst:
58 //-----------------------------------------------------------------------------
60 //-----------------------------------------------------------------------------
61 static inline void thread_fence(memory_order order) CDS_NOEXCEPT
65 case memory_order_relaxed:
66 case memory_order_consume:
68 case memory_order_release:
69 case memory_order_acquire:
70 case memory_order_acq_rel:
71 CDS_COMPILER_RW_BARRIER;
73 case memory_order_seq_cst:
80 static inline void signal_fence(memory_order order) CDS_NOEXCEPT
82 // C++11: 29.8.8: only compiler optimization, no hardware instructions
85 case memory_order_relaxed:
87 case memory_order_consume:
88 case memory_order_release:
89 case memory_order_acquire:
90 case memory_order_acq_rel:
91 case memory_order_seq_cst:
92 CDS_COMPILER_RW_BARRIER;
98 #define CDS_ITANIUM_ATOMIC_LOAD( n_bytes, n_bits ) \
99 template <typename T> \
100 static inline T load##n_bits( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT \
102 static_assert( sizeof(T) == n_bytes, "Illegal size of operand" ) ; \
103 assert( order == memory_order_relaxed \
104 || order == memory_order_consume \
105 || order == memory_order_acquire \
106 || order == memory_order_seq_cst \
110 __asm__ __volatile__ ( \
111 "ld" #n_bytes ".acq %[val] = [%[pSrc]] \n\t" \
113 : [pSrc] "r" (pSrc) \
119 #define CDS_ITANIUM_ATOMIC_STORE( n_bytes, n_bits ) \
120 template <typename T> \
121 static inline void store##n_bits( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT \
123 static_assert( sizeof(T) == n_bytes, "Illegal size of operand" ) ; \
124 assert( order == memory_order_relaxed \
125 || order == memory_order_release \
126 || order == memory_order_seq_cst \
129 if ( order == memory_order_seq_cst ) { \
130 __asm__ __volatile__ ( \
131 "st" #n_bytes ".rel [%[pDest]] = %[val] \n\t" \
133 :: [pDest] "r" (pDest), [val] "r" (val) \
138 __asm__ __volatile__ ( \
139 "st" #n_bytes ".rel [%[pDest]] = %[val] \n\t" \
140 :: [pDest] "r" (pDest), [val] "r" (val) \
143 fence_after(order) ; \
147 #define CDS_ITANIUM_ATOMIC_CAS( n_bytes, n_bits ) \
148 template <typename T> \
149 static inline bool cas##n_bits##_strong( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order /*mo_fail*/ ) CDS_NOEXCEPT \
151 static_assert( sizeof(T) == n_bytes, "Illegal size of operand" ) ; \
153 switch(mo_success) { \
154 case memory_order_relaxed: \
155 case memory_order_consume: \
156 case memory_order_acquire: \
157 __asm__ __volatile__ ( \
158 "mov ar.ccv = %[expected] ;;\n\t" \
159 "cmpxchg" #n_bytes ".acq %[current] = [%[pDest]], %[desired], ar.ccv\n\t" \
160 : [current] "=r" (current) \
161 : [pDest] "r" (pDest), [expected] "r" (expected), [desired] "r" (desired) \
162 : "ar.ccv", "memory" \
165 case memory_order_release: \
166 __asm__ __volatile__ ( \
167 "mov ar.ccv = %[expected] ;;\n\t" \
168 "cmpxchg" #n_bytes ".rel %[current] = [%[pDest]], %[desired], ar.ccv\n\t" \
169 : [current] "=r" (current) \
170 : [pDest] "r" (pDest), [expected] "r" (expected), [desired] "r" (desired) \
171 : "ar.ccv", "memory" \
174 case memory_order_acq_rel: \
175 case memory_order_seq_cst: \
176 __asm__ __volatile__ ( \
177 "mov ar.ccv = %[expected] ;;\n\t" \
178 "cmpxchg" #n_bytes ".rel %[current] = [%[pDest]], %[desired], ar.ccv\n\t" \
180 : [current] "=r" (current) \
181 : [pDest] "r" (pDest), [expected] "r" (expected), [desired] "r" (desired) \
182 : "ar.ccv", "memory" \
188 bool bSuccess = expected == current ; \
189 expected = current ; \
192 template <typename T> \
193 static inline bool cas##n_bits##_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT \
194 { return cas##n_bits##_strong( pDest, expected, desired, mo_success, mo_fail ); }
196 // xchg is performed with acquire semantics
197 #define CDS_ITANIUM_ATOMIC_EXCHANGE( n_bytes, n_bits ) \
198 template <typename T> \
199 static inline T exchange##n_bits( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT \
201 static_assert( sizeof(T) == n_bytes, "Illegal size of operand" ) ; \
206 case memory_order_relaxed: \
207 case memory_order_consume: \
208 case memory_order_acquire: \
209 __asm__ __volatile__ ( \
210 "xchg" #n_bytes " %[current] = [%[pDest]], %[val]\n\t" \
211 : [current] "=r" (current) \
212 : [pDest] "r" (pDest), [val] "r" (val) \
216 case memory_order_acq_rel: \
217 case memory_order_release: \
218 case memory_order_seq_cst: \
219 __asm__ __volatile__ ( \
221 "xchg" #n_bytes " %[current] = [%[pDest]], %[val]\n\t" \
222 : [current] "=r" (current) \
223 : [pDest] "r" (pDest), [val] "r" (val) \
227 default: assert(false); \
232 #define CDS_ITANIUM_ATOMIC_FETCH_ADD( n_bytes, n_add ) \
234 case memory_order_relaxed: \
235 case memory_order_consume: \
236 case memory_order_acquire: \
237 __asm__ __volatile__ ( \
238 "fetchadd" #n_bytes ".acq %[cur] = [%[pDest]], " #n_add " \n\t" \
240 : [pDest] "r" (pDest) \
244 case memory_order_release: \
245 __asm__ __volatile__ ( \
246 "fetchadd" #n_bytes ".rel %[cur] = [%[pDest]], " #n_add " \n\t" \
248 : [pDest] "r" (pDest) \
252 case memory_order_acq_rel: \
253 case memory_order_seq_cst: \
254 __asm__ __volatile__ ( \
255 "fetchadd" #n_bytes ".rel %[cur] = [%[pDest]], " #n_add " \n\t" \
258 : [pDest] "r" (pDest) \
266 //-----------------------------------------------------------------------------
268 //-----------------------------------------------------------------------------
270 CDS_ITANIUM_ATOMIC_LOAD( 1, 8 )
271 CDS_ITANIUM_ATOMIC_STORE( 1, 8 )
272 CDS_ITANIUM_ATOMIC_CAS( 1, 8 )
273 CDS_ITANIUM_ATOMIC_EXCHANGE( 1, 8 )
275 //-----------------------------------------------------------------------------
277 //-----------------------------------------------------------------------------
279 CDS_ITANIUM_ATOMIC_LOAD( 2, 16 )
280 CDS_ITANIUM_ATOMIC_STORE( 2, 16 )
281 CDS_ITANIUM_ATOMIC_CAS( 2, 16 )
282 CDS_ITANIUM_ATOMIC_EXCHANGE( 2, 16 )
284 //-----------------------------------------------------------------------------
286 //-----------------------------------------------------------------------------
288 CDS_ITANIUM_ATOMIC_LOAD( 4, 32 )
289 CDS_ITANIUM_ATOMIC_STORE( 4, 32 )
290 CDS_ITANIUM_ATOMIC_CAS( 4, 32 )
291 CDS_ITANIUM_ATOMIC_EXCHANGE( 4, 32 )
293 # define CDS_ATOMIC_fetch32_add_defined
294 template <typename T>
295 static inline T fetch32_add( T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
297 static_assert( sizeof(T) == 4, "Illegal size of operand" );
303 CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, 1 );
306 CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, 4 );
309 CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, 8 );
312 CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, 16 );
315 cur = load32( pDest, memory_order_relaxed );
316 do {} while ( !cas32_strong( pDest, cur, cur + val, order, memory_order_relaxed ));
322 # define CDS_ATOMIC_fetch32_sub_defined
323 template <typename T>
324 static inline T fetch32_sub( T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
326 static_assert( sizeof(T) == 4, "Illegal size of operand" );
331 CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, -1 );
334 CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, -4 );
337 CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, -8 );
340 CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, -16 );
343 cur = load32( pDest, memory_order_relaxed );
344 do {} while ( !cas32_strong( pDest, cur, cur - val, order, memory_order_relaxed ));
350 //-----------------------------------------------------------------------------
352 //-----------------------------------------------------------------------------
354 CDS_ITANIUM_ATOMIC_LOAD( 8, 64 )
355 CDS_ITANIUM_ATOMIC_STORE( 8, 64 )
356 CDS_ITANIUM_ATOMIC_CAS( 8, 64 )
357 CDS_ITANIUM_ATOMIC_EXCHANGE( 8, 64 )
359 # define CDS_ATOMIC_fetch64_add_defined
360 template <typename T>
361 static inline T fetch64_add( T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
363 static_assert( sizeof(T) == 8, "Illegal size of operand" );
369 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 1 );
372 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 4 );
375 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 8 );
378 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 16 );
381 cur = load64( pDest, memory_order_relaxed );
382 do {} while ( !cas64_strong( pDest, cur, cur + val, order, memory_order_relaxed ));
388 # define CDS_ATOMIC_fetch64_sub_defined
389 template <typename T>
390 static inline T fetch64_sub( T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
392 static_assert( sizeof(T) == 8, "Illegal size of operand" );
397 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -1 );
400 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -4 );
403 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -8 );
406 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -16 );
409 cur = load64( pDest, memory_order_relaxed );
410 do {} while ( !cas64_strong( pDest, cur, cur - val, order, memory_order_relaxed ));
416 //-----------------------------------------------------------------------------
417 // pointer primitives
418 //-----------------------------------------------------------------------------
419 template <typename T>
420 static inline T * load_ptr( T * volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
422 assert( order == memory_order_relaxed
423 || order == memory_order_consume
424 || order == memory_order_acquire
425 || order == memory_order_seq_cst
429 __asm__ __volatile__ (
430 "ld8.acq %[val] = [%[pSrc]] \n\t"
438 template <typename T>
439 static inline void store_ptr( T * volatile * pDest, T * val, memory_order order ) CDS_NOEXCEPT
441 assert( order == memory_order_relaxed
442 || order == memory_order_release
443 || order == memory_order_seq_cst
447 if ( order == memory_order_seq_cst ) {
448 __asm__ __volatile__ (
449 "st8.rel [%[pDest]] = %[val] \n\t"
451 :: [pDest] "r" (pDest), [val] "r" (val)
456 __asm__ __volatile__ (
457 "st8.rel [%[pDest]] = %[val] \n\t"
458 :: [pDest] "r" (pDest), [val] "r" (val)
465 template <typename T>
466 static inline bool cas_ptr_strong( T * volatile * pDest, T *& expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
468 static_assert( sizeof(T *) == 8, "Illegal size of operand" );
474 case memory_order_relaxed:
475 case memory_order_consume:
476 case memory_order_acquire:
477 __asm__ __volatile__ (
478 "mov ar.ccv = %[expected] ;;\n\t"
479 "cmpxchg8.acq %[current] = [%[pDest]], %[desired], ar.ccv\n\t"
480 : [current] "=r" (current)
481 : [pDest] "r" (pDest), [expected] "r" (expected), [desired] "r" (desired)
485 case memory_order_release:
486 __asm__ __volatile__ (
487 "mov ar.ccv = %[expected] ;;\n\t"
488 "cmpxchg8.rel %[current] = [%[pDest]], %[desired], ar.ccv\n\t"
489 : [current] "=r" (current)
490 : [pDest] "r" (pDest), [expected] "r" (expected), [desired] "r" (desired)
494 case memory_order_acq_rel:
495 case memory_order_seq_cst:
496 __asm__ __volatile__ (
497 "mov ar.ccv = %[expected] ;;\n\t"
498 "cmpxchg8.rel %[current] = [%[pDest]], %[desired], ar.ccv\n\t"
500 : [current] "=r" (current)
501 : [pDest] "r" (pDest), [expected] "r" (expected), [desired] "r" (desired)
509 bool bSuccess = expected == current;
512 fence_after( mo_fail );
516 template <typename T>
517 static inline bool cas_ptr_weak( T * volatile * pDest, T *& expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
519 return cas_ptr_strong( pDest, expected, desired, mo_success, mo_fail );
522 template <typename T>
523 static inline T * exchange_ptr( T * volatile * pDest, T * val, memory_order order ) CDS_NOEXCEPT
525 static_assert( sizeof(T *) == 8, "Illegal size of operand" );
530 case memory_order_relaxed:
531 case memory_order_consume:
532 case memory_order_acquire:
533 __asm__ __volatile__ (
534 "xchg8 %[current] = [%[pDest]], %[val]\n\t"
535 : [current] "=r" (current)
536 : [pDest] "r" (pDest), [val] "r" (val)
540 case memory_order_acq_rel:
541 case memory_order_release:
542 case memory_order_seq_cst:
543 __asm__ __volatile__ (
545 "xchg8 %[current] = [%[pDest]], %[val]\n\t"
546 : [current] "=r" (current)
547 : [pDest] "r" (pDest), [val] "r" (val)
551 default: assert(false);
557 template <typename T> struct atomic_pointer_sizeof { enum { value = sizeof(T) }; };
558 template <> struct atomic_pointer_sizeof<void> { enum { value = 1 }; };
560 // It does not work properly
561 // atomic.fetch_add( ... ) returns nullptr, why?..
562 //# define CDS_ATOMIC_fetch_ptr_add_defined
563 template <typename T>
564 static inline T * fetch_ptr_add( T * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
566 static_assert( sizeof(T *) == 8, "Illegal size of operand" );
570 val *= atomic_pointer_sizeof<T>::value;
573 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 1 );
576 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 4 );
579 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 8 );
582 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 16 );
585 cur = load_ptr( pDest, memory_order_relaxed );
586 do {} while ( !cas_ptr_strong( pDest, cur, reinterpret_cast<T *>(reinterpret_cast<uint8_t *>(cur) + val), order, memory_order_relaxed ));
592 // It does not work properly
593 // atomic.fetch_sub( ... ) returns nullptr, why?..
594 //# define CDS_ATOMIC_fetch_ptr_sub_defined
595 template <typename T>
596 static inline T * fetch_ptr_sub( T * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
598 static_assert( sizeof(T *) == 8, "Illegal size of operand" );
601 val *= atomic_pointer_sizeof<T>::value;
604 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -1 );
607 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -4 );
610 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -8 );
613 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -16 );
616 cur = load_ptr( pDest, memory_order_relaxed );
617 do {} while ( !cas_ptr_strong( pDest, cur, reinterpret_cast<T *>(reinterpret_cast<uint8_t *>(cur) - val), order, memory_order_relaxed ));
623 //-----------------------------------------------------------------------------
624 // atomic flag primitives
625 //-----------------------------------------------------------------------------
627 typedef bool atomic_flag_type;
628 static inline bool atomic_flag_tas( atomic_flag_type volatile * pFlag, memory_order order ) CDS_NOEXCEPT
630 return exchange8( pFlag, true, order );
633 static inline void atomic_flag_clear( atomic_flag_type volatile * pFlag, memory_order order ) CDS_NOEXCEPT
635 store8( pFlag, false, order );
638 #undef CDS_ITANIUM_ATOMIC_LOAD
639 #undef CDS_ITANIUM_ATOMIC_STORE
640 #undef CDS_ITANIUM_ATOMIC_CAS
641 #undef CDS_ITANIUM_ATOMIC_EXCHANGE
642 #undef CDS_ITANIUM_ATOMIC_FETCH_ADD
644 }} // namespace gcc::ia64
646 #ifndef CDS_CXX11_INLINE_NAMESPACE_SUPPORT
647 using namespace gcc::ia64;
649 } // namespace platform
650 }} // namespace cds::cxx11_atomic
653 #endif // #ifndef __CDS_COMPILER_GCC_IA64_CXX11_ATOMIC_H