2 This file is a part of libcds - Concurrent Data Structures library
4 (C) Copyright Maxim Khizhinsky (libcds.dev@gmail.com) 2006-2017
6 Source code repo: http://github.com/khizmax/libcds/
7 Download: http://sourceforge.net/projects/libcds/files/
9 Redistribution and use in source and binary forms, with or without
10 modification, are permitted provided that the following conditions are met:
12 * Redistributions of source code must retain the above copyright notice, this
13 list of conditions and the following disclaimer.
15 * Redistributions in binary form must reproduce the above copyright notice,
16 this list of conditions and the following disclaimer in the documentation
17 and/or other materials provided with the distribution.
19 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
23 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
26 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 #ifndef CDSLIB_COMPILER_GCC_IA64_CXX11_ATOMIC_H
32 #define CDSLIB_COMPILER_GCC_IA64_CXX11_ATOMIC_H
36 1. load/store: http://www.decadent.org.uk/pipermail/cpp-threads/2008-December/001932.html
37 2. Mapping to C++ Memory Model: http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
43 namespace cds { namespace cxx11_atomic {
44 namespace platform { inline namespace gcc { inline namespace ia64 {
46 static inline void itanium_full_fence() noexcept
48 __asm__ __volatile__ ( "mf \n\t" ::: "memory" );
51 static inline void fence_before( memory_order order ) noexcept
54 case memory_order_relaxed:
55 case memory_order_consume:
56 case memory_order_acquire:
58 case memory_order_release:
59 case memory_order_acq_rel:
60 CDS_COMPILER_RW_BARRIER;
62 case memory_order_seq_cst:
68 static inline void fence_after( memory_order order ) noexcept
71 case memory_order_acquire:
72 case memory_order_acq_rel:
73 CDS_COMPILER_RW_BARRIER;
75 case memory_order_relaxed:
76 case memory_order_consume:
77 case memory_order_release:
79 case memory_order_seq_cst:
86 //-----------------------------------------------------------------------------
88 //-----------------------------------------------------------------------------
89 static inline void thread_fence(memory_order order) noexcept
93 case memory_order_relaxed:
94 case memory_order_consume:
96 case memory_order_release:
97 case memory_order_acquire:
98 case memory_order_acq_rel:
99 CDS_COMPILER_RW_BARRIER;
101 case memory_order_seq_cst:
102 itanium_full_fence();
108 static inline void signal_fence(memory_order order) noexcept
110 // C++11: 29.8.8: only compiler optimization, no hardware instructions
113 case memory_order_relaxed:
115 case memory_order_consume:
116 case memory_order_release:
117 case memory_order_acquire:
118 case memory_order_acq_rel:
119 case memory_order_seq_cst:
120 CDS_COMPILER_RW_BARRIER;
126 #define CDS_ITANIUM_ATOMIC_LOAD( n_bytes, n_bits ) \
127 template <typename T> \
128 static inline T load##n_bits( T volatile const * pSrc, memory_order order ) noexcept \
130 static_assert( sizeof(T) == n_bytes, "Illegal size of operand" ) ; \
131 assert( order == memory_order_relaxed \
132 || order == memory_order_consume \
133 || order == memory_order_acquire \
134 || order == memory_order_seq_cst \
138 __asm__ __volatile__ ( \
139 "ld" #n_bytes ".acq %[val] = [%[pSrc]] \n\t" \
141 : [pSrc] "r" (pSrc) \
147 #define CDS_ITANIUM_ATOMIC_STORE( n_bytes, n_bits ) \
148 template <typename T> \
149 static inline void store##n_bits( T volatile * pDest, T val, memory_order order ) noexcept \
151 static_assert( sizeof(T) == n_bytes, "Illegal size of operand" ) ; \
152 assert( order == memory_order_relaxed \
153 || order == memory_order_release \
154 || order == memory_order_seq_cst \
157 if ( order == memory_order_seq_cst ) { \
158 __asm__ __volatile__ ( \
159 "st" #n_bytes ".rel [%[pDest]] = %[val] \n\t" \
161 :: [pDest] "r" (pDest), [val] "r" (val) \
166 __asm__ __volatile__ ( \
167 "st" #n_bytes ".rel [%[pDest]] = %[val] \n\t" \
168 :: [pDest] "r" (pDest), [val] "r" (val) \
171 fence_after(order) ; \
175 #define CDS_ITANIUM_ATOMIC_CAS( n_bytes, n_bits ) \
176 template <typename T> \
177 static inline bool cas##n_bits##_strong( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order /*mo_fail*/ ) noexcept \
179 static_assert( sizeof(T) == n_bytes, "Illegal size of operand" ) ; \
181 switch(mo_success) { \
182 case memory_order_relaxed: \
183 case memory_order_consume: \
184 case memory_order_acquire: \
185 __asm__ __volatile__ ( \
186 "mov ar.ccv = %[expected] ;;\n\t" \
187 "cmpxchg" #n_bytes ".acq %[current] = [%[pDest]], %[desired], ar.ccv\n\t" \
188 : [current] "=r" (current) \
189 : [pDest] "r" (pDest), [expected] "r" (expected), [desired] "r" (desired) \
190 : "ar.ccv", "memory" \
193 case memory_order_release: \
194 __asm__ __volatile__ ( \
195 "mov ar.ccv = %[expected] ;;\n\t" \
196 "cmpxchg" #n_bytes ".rel %[current] = [%[pDest]], %[desired], ar.ccv\n\t" \
197 : [current] "=r" (current) \
198 : [pDest] "r" (pDest), [expected] "r" (expected), [desired] "r" (desired) \
199 : "ar.ccv", "memory" \
202 case memory_order_acq_rel: \
203 case memory_order_seq_cst: \
204 __asm__ __volatile__ ( \
205 "mov ar.ccv = %[expected] ;;\n\t" \
206 "cmpxchg" #n_bytes ".rel %[current] = [%[pDest]], %[desired], ar.ccv\n\t" \
208 : [current] "=r" (current) \
209 : [pDest] "r" (pDest), [expected] "r" (expected), [desired] "r" (desired) \
210 : "ar.ccv", "memory" \
216 bool bSuccess = expected == current ; \
217 expected = current ; \
220 template <typename T> \
221 static inline bool cas##n_bits##_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) noexcept \
222 { return cas##n_bits##_strong( pDest, expected, desired, mo_success, mo_fail ); }
224 // xchg is performed with acquire semantics
225 #define CDS_ITANIUM_ATOMIC_EXCHANGE( n_bytes, n_bits ) \
226 template <typename T> \
227 static inline T exchange##n_bits( T volatile * pDest, T val, memory_order order ) noexcept \
229 static_assert( sizeof(T) == n_bytes, "Illegal size of operand" ) ; \
234 case memory_order_relaxed: \
235 case memory_order_consume: \
236 case memory_order_acquire: \
237 __asm__ __volatile__ ( \
238 "xchg" #n_bytes " %[current] = [%[pDest]], %[val]\n\t" \
239 : [current] "=r" (current) \
240 : [pDest] "r" (pDest), [val] "r" (val) \
244 case memory_order_acq_rel: \
245 case memory_order_release: \
246 case memory_order_seq_cst: \
247 __asm__ __volatile__ ( \
249 "xchg" #n_bytes " %[current] = [%[pDest]], %[val]\n\t" \
250 : [current] "=r" (current) \
251 : [pDest] "r" (pDest), [val] "r" (val) \
255 default: assert(false); \
260 #define CDS_ITANIUM_ATOMIC_FETCH_ADD( n_bytes, n_add ) \
262 case memory_order_relaxed: \
263 case memory_order_consume: \
264 case memory_order_acquire: \
265 __asm__ __volatile__ ( \
266 "fetchadd" #n_bytes ".acq %[cur] = [%[pDest]], " #n_add " \n\t" \
268 : [pDest] "r" (pDest) \
272 case memory_order_release: \
273 __asm__ __volatile__ ( \
274 "fetchadd" #n_bytes ".rel %[cur] = [%[pDest]], " #n_add " \n\t" \
276 : [pDest] "r" (pDest) \
280 case memory_order_acq_rel: \
281 case memory_order_seq_cst: \
282 __asm__ __volatile__ ( \
283 "fetchadd" #n_bytes ".rel %[cur] = [%[pDest]], " #n_add " \n\t" \
286 : [pDest] "r" (pDest) \
294 //-----------------------------------------------------------------------------
296 //-----------------------------------------------------------------------------
298 CDS_ITANIUM_ATOMIC_LOAD( 1, 8 )
299 CDS_ITANIUM_ATOMIC_STORE( 1, 8 )
300 CDS_ITANIUM_ATOMIC_CAS( 1, 8 )
301 CDS_ITANIUM_ATOMIC_EXCHANGE( 1, 8 )
303 //-----------------------------------------------------------------------------
305 //-----------------------------------------------------------------------------
307 CDS_ITANIUM_ATOMIC_LOAD( 2, 16 )
308 CDS_ITANIUM_ATOMIC_STORE( 2, 16 )
309 CDS_ITANIUM_ATOMIC_CAS( 2, 16 )
310 CDS_ITANIUM_ATOMIC_EXCHANGE( 2, 16 )
312 //-----------------------------------------------------------------------------
314 //-----------------------------------------------------------------------------
316 CDS_ITANIUM_ATOMIC_LOAD( 4, 32 )
317 CDS_ITANIUM_ATOMIC_STORE( 4, 32 )
318 CDS_ITANIUM_ATOMIC_CAS( 4, 32 )
319 CDS_ITANIUM_ATOMIC_EXCHANGE( 4, 32 )
321 # define CDS_ATOMIC_fetch32_add_defined
322 template <typename T>
323 static inline T fetch32_add( T volatile * pDest, T val, memory_order order) noexcept
325 static_assert( sizeof(T) == 4, "Illegal size of operand" );
331 CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, 1 );
334 CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, 4 );
337 CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, 8 );
340 CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, 16 );
343 cur = load32( pDest, memory_order_relaxed );
344 do {} while ( !cas32_strong( pDest, cur, cur + val, order, memory_order_relaxed ));
350 # define CDS_ATOMIC_fetch32_sub_defined
351 template <typename T>
352 static inline T fetch32_sub( T volatile * pDest, T val, memory_order order) noexcept
354 static_assert( sizeof(T) == 4, "Illegal size of operand" );
359 CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, -1 );
362 CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, -4 );
365 CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, -8 );
368 CDS_ITANIUM_ATOMIC_FETCH_ADD( 4, -16 );
371 cur = load32( pDest, memory_order_relaxed );
372 do {} while ( !cas32_strong( pDest, cur, cur - val, order, memory_order_relaxed ));
378 //-----------------------------------------------------------------------------
380 //-----------------------------------------------------------------------------
382 CDS_ITANIUM_ATOMIC_LOAD( 8, 64 )
383 CDS_ITANIUM_ATOMIC_STORE( 8, 64 )
384 CDS_ITANIUM_ATOMIC_CAS( 8, 64 )
385 CDS_ITANIUM_ATOMIC_EXCHANGE( 8, 64 )
387 # define CDS_ATOMIC_fetch64_add_defined
388 template <typename T>
389 static inline T fetch64_add( T volatile * pDest, T val, memory_order order) noexcept
391 static_assert( sizeof(T) == 8, "Illegal size of operand" );
397 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 1 );
400 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 4 );
403 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 8 );
406 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 16 );
409 cur = load64( pDest, memory_order_relaxed );
410 do {} while ( !cas64_strong( pDest, cur, cur + val, order, memory_order_relaxed ));
416 # define CDS_ATOMIC_fetch64_sub_defined
417 template <typename T>
418 static inline T fetch64_sub( T volatile * pDest, T val, memory_order order) noexcept
420 static_assert( sizeof(T) == 8, "Illegal size of operand" );
425 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -1 );
428 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -4 );
431 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -8 );
434 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -16 );
437 cur = load64( pDest, memory_order_relaxed );
438 do {} while ( !cas64_strong( pDest, cur, cur - val, order, memory_order_relaxed ));
444 //-----------------------------------------------------------------------------
445 // pointer primitives
446 //-----------------------------------------------------------------------------
447 template <typename T>
448 static inline T * load_ptr( T * volatile const * pSrc, memory_order order ) noexcept
450 assert( order == memory_order_relaxed
451 || order == memory_order_consume
452 || order == memory_order_acquire
453 || order == memory_order_seq_cst
457 __asm__ __volatile__ (
458 "ld8.acq %[val] = [%[pSrc]] \n\t"
466 template <typename T>
467 static inline void store_ptr( T * volatile * pDest, T * val, memory_order order ) noexcept
469 assert( order == memory_order_relaxed
470 || order == memory_order_release
471 || order == memory_order_seq_cst
475 if ( order == memory_order_seq_cst ) {
476 __asm__ __volatile__ (
477 "st8.rel [%[pDest]] = %[val] \n\t"
479 :: [pDest] "r" (pDest), [val] "r" (val)
484 __asm__ __volatile__ (
485 "st8.rel [%[pDest]] = %[val] \n\t"
486 :: [pDest] "r" (pDest), [val] "r" (val)
493 template <typename T>
494 static inline bool cas_ptr_strong( T * volatile * pDest, T *& expected, T * desired, memory_order mo_success, memory_order mo_fail ) noexcept
496 static_assert( sizeof(T *) == 8, "Illegal size of operand" );
502 case memory_order_relaxed:
503 case memory_order_consume:
504 case memory_order_acquire:
505 __asm__ __volatile__ (
506 "mov ar.ccv = %[expected] ;;\n\t"
507 "cmpxchg8.acq %[current] = [%[pDest]], %[desired], ar.ccv\n\t"
508 : [current] "=r" (current)
509 : [pDest] "r" (pDest), [expected] "r" (expected), [desired] "r" (desired)
513 case memory_order_release:
514 __asm__ __volatile__ (
515 "mov ar.ccv = %[expected] ;;\n\t"
516 "cmpxchg8.rel %[current] = [%[pDest]], %[desired], ar.ccv\n\t"
517 : [current] "=r" (current)
518 : [pDest] "r" (pDest), [expected] "r" (expected), [desired] "r" (desired)
522 case memory_order_acq_rel:
523 case memory_order_seq_cst:
524 __asm__ __volatile__ (
525 "mov ar.ccv = %[expected] ;;\n\t"
526 "cmpxchg8.rel %[current] = [%[pDest]], %[desired], ar.ccv\n\t"
528 : [current] "=r" (current)
529 : [pDest] "r" (pDest), [expected] "r" (expected), [desired] "r" (desired)
537 bool bSuccess = expected == current;
540 fence_after( mo_fail );
544 template <typename T>
545 static inline bool cas_ptr_weak( T * volatile * pDest, T *& expected, T * desired, memory_order mo_success, memory_order mo_fail ) noexcept
547 return cas_ptr_strong( pDest, expected, desired, mo_success, mo_fail );
550 template <typename T>
551 static inline T * exchange_ptr( T * volatile * pDest, T * val, memory_order order ) noexcept
553 static_assert( sizeof(T *) == 8, "Illegal size of operand" );
558 case memory_order_relaxed:
559 case memory_order_consume:
560 case memory_order_acquire:
561 __asm__ __volatile__ (
562 "xchg8 %[current] = [%[pDest]], %[val]\n\t"
563 : [current] "=r" (current)
564 : [pDest] "r" (pDest), [val] "r" (val)
568 case memory_order_acq_rel:
569 case memory_order_release:
570 case memory_order_seq_cst:
571 __asm__ __volatile__ (
573 "xchg8 %[current] = [%[pDest]], %[val]\n\t"
574 : [current] "=r" (current)
575 : [pDest] "r" (pDest), [val] "r" (val)
579 default: assert(false);
585 template <typename T> struct atomic_pointer_sizeof { enum { value = sizeof(T) }; };
586 template <> struct atomic_pointer_sizeof<void> { enum { value = 1 }; };
588 // It does not work properly
589 // atomic.fetch_add( ... ) returns nullptr, why?..
590 //# define CDS_ATOMIC_fetch_ptr_add_defined
591 template <typename T>
592 static inline T * fetch_ptr_add( T * volatile * pDest, ptrdiff_t val, memory_order order) noexcept
594 static_assert( sizeof(T *) == 8, "Illegal size of operand" );
598 val *= atomic_pointer_sizeof<T>::value;
601 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 1 );
604 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 4 );
607 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 8 );
610 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, 16 );
613 cur = load_ptr( pDest, memory_order_relaxed );
614 do {} while ( !cas_ptr_strong( pDest, cur, reinterpret_cast<T *>(reinterpret_cast<uint8_t *>(cur) + val), order, memory_order_relaxed ));
620 // It does not work properly
621 // atomic.fetch_sub( ... ) returns nullptr, why?..
622 //# define CDS_ATOMIC_fetch_ptr_sub_defined
623 template <typename T>
624 static inline T * fetch_ptr_sub( T * volatile * pDest, ptrdiff_t val, memory_order order) noexcept
626 static_assert( sizeof(T *) == 8, "Illegal size of operand" );
629 val *= atomic_pointer_sizeof<T>::value;
632 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -1 );
635 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -4 );
638 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -8 );
641 CDS_ITANIUM_ATOMIC_FETCH_ADD( 8, -16 );
644 cur = load_ptr( pDest, memory_order_relaxed );
645 do {} while ( !cas_ptr_strong( pDest, cur, reinterpret_cast<T *>(reinterpret_cast<uint8_t *>(cur) - val), order, memory_order_relaxed ));
651 //-----------------------------------------------------------------------------
652 // atomic flag primitives
653 //-----------------------------------------------------------------------------
655 typedef bool atomic_flag_type;
656 static inline bool atomic_flag_tas( atomic_flag_type volatile * pFlag, memory_order order ) noexcept
658 return exchange8( pFlag, true, order );
661 static inline void atomic_flag_clear( atomic_flag_type volatile * pFlag, memory_order order ) noexcept
663 store8( pFlag, false, order );
666 #undef CDS_ITANIUM_ATOMIC_LOAD
667 #undef CDS_ITANIUM_ATOMIC_STORE
668 #undef CDS_ITANIUM_ATOMIC_CAS
669 #undef CDS_ITANIUM_ATOMIC_EXCHANGE
670 #undef CDS_ITANIUM_ATOMIC_FETCH_ADD
672 }} // namespace gcc::ia64
674 } // namespace platform
675 }} // namespace cds::cxx11_atomic
678 #endif // #ifndef CDSLIB_COMPILER_GCC_IA64_CXX11_ATOMIC_H