3 #ifndef CDSLIB_COMPILER_CXX11_ATOMIC_H
4 #define CDSLIB_COMPILER_CXX11_ATOMIC_H
7 #include <type_traits> // make_unsigned
8 #include <cds/details/defs.h>
9 #include <cds/details/aligned_type.h>
11 namespace cds { namespace cxx11_atomic {
12 typedef enum memory_order {
21 }} // namespace cds::cxx11_atomic
24 #if CDS_COMPILER == CDS_COMPILER_MSVC || (CDS_COMPILER == CDS_COMPILER_INTEL && CDS_OS_INTERFACE == CDS_OSI_WINDOWS)
25 # if CDS_PROCESSOR_ARCH == CDS_PROCESSOR_X86
26 # include <cds/compiler/vc/x86/cxx11_atomic.h>
27 # elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_AMD64
28 # include <cds/compiler/vc/amd64/cxx11_atomic.h>
30 # error "MS VC++ compiler: unsupported processor architecture"
32 #elif CDS_COMPILER == CDS_COMPILER_GCC || CDS_COMPILER == CDS_COMPILER_CLANG || CDS_COMPILER == CDS_COMPILER_INTEL
33 # if CDS_PROCESSOR_ARCH == CDS_PROCESSOR_X86
34 # include <cds/compiler/gcc/x86/cxx11_atomic.h>
35 # elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_AMD64
36 # include <cds/compiler/gcc/amd64/cxx11_atomic.h>
37 # elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_IA64
38 # include <cds/compiler/gcc/ia64/cxx11_atomic.h>
39 # elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_SPARC
40 # include <cds/compiler/gcc/sparc/cxx11_atomic.h>
41 # elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_PPC64
42 # include <cds/compiler/gcc/ppc64/cxx11_atomic.h>
43 //# elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_ARM7
44 //# include <cds/compiler/gcc/arm7/cxx11_atomic.h>
46 # error "GCC compiler: unsupported processor architecture. Try to use native C++11 atomic or boost.atomic"
49 # error "Undefined compiler"
52 namespace cds { namespace cxx11_atomic {
54 // forward declarations
60 template <typename T, size_t Size, typename Primary = T >
61 struct atomic_generic_ops;
63 template <typename T, size_t Size>
64 struct atomic_integral_ops;
66 template <size_t TypeSize>
70 struct primary_type<1>
72 typedef std::uint8_t type;
75 struct primary_type<2>
77 typedef std::uint16_t type;
80 struct primary_type<4>
82 typedef std::uint32_t type;
85 struct primary_type<8>
87 typedef std::uint64_t type;
90 template <typename T, typename Primary>
91 struct make_atomic_primary
93 typedef T source_type;
94 typedef Primary primary_type;
96 static primary_type volatile * ptr( source_type volatile * p ) CDS_NOEXCEPT
98 return reinterpret_cast<primary_type volatile *>(p);
100 static primary_type const volatile * ptr( source_type const volatile * p ) CDS_NOEXCEPT
102 return reinterpret_cast<primary_type const volatile *>(p);
105 static primary_type val( source_type v ) CDS_NOEXCEPT
107 return *reinterpret_cast<primary_type*>(&v);
110 static primary_type& ref( source_type& v ) CDS_NOEXCEPT
112 return reinterpret_cast<primary_type&>(v);
115 static primary_type const& ref( source_type const& v ) CDS_NOEXCEPT
117 return reinterpret_cast<primary_type const&>(v);
120 static source_type ret( primary_type r ) CDS_NOEXCEPT
122 return *reinterpret_cast<source_type *>(&r);
126 template <typename T>
127 struct make_atomic_primary<T, T>
129 typedef T source_type;
130 typedef T primary_type;
132 static primary_type volatile * ptr( source_type volatile * p ) CDS_NOEXCEPT
136 static primary_type const volatile * ptr( source_type const volatile * p ) CDS_NOEXCEPT
141 static primary_type val( source_type v ) CDS_NOEXCEPT
146 static primary_type& ref( source_type& v ) CDS_NOEXCEPT
151 static source_type ret( primary_type r ) CDS_NOEXCEPT
157 template <typename T>
158 struct atomic_integral_bitwise_ops
161 typedef typename std::make_unsigned<T>::type unsigned_type;
162 typedef atomic_generic_ops<unsigned_type, sizeof(unsigned_type)> atomic_ops;
164 static T fetch_and(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
166 unsigned_type cur = atomic_ops::atomic_load_explicit( reinterpret_cast<unsigned_type volatile *>(pDest), memory_order_relaxed );
167 do {} while ( !atomic_ops::atomic_compare_exchange_weak_explicit(
168 reinterpret_cast<unsigned_type volatile *>(pDest), &cur, cur & unsigned_type(val), order, memory_order_relaxed ));
172 static T fetch_or(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
174 unsigned_type cur = atomic_ops::atomic_load_explicit( reinterpret_cast<unsigned_type volatile *>(pDest), memory_order_relaxed );
175 do {} while ( !atomic_ops::atomic_compare_exchange_weak_explicit(
176 reinterpret_cast<unsigned_type volatile *>(pDest), &cur, cur | unsigned_type(val), order, memory_order_relaxed ));
180 static T fetch_xor(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
182 unsigned_type cur = atomic_ops::atomic_load_explicit( reinterpret_cast<unsigned_type volatile *>(pDest), memory_order_relaxed );
183 do {} while ( !atomic_ops::atomic_compare_exchange_weak_explicit(
184 reinterpret_cast<unsigned_type volatile *>(pDest), &cur, cur ^ unsigned_type(val), order, memory_order_relaxed ));
190 // 8-bit atomic operations
192 template <typename T, typename Primary>
193 struct atomic_generic_ops< T, 1, Primary >
195 typedef make_atomic_primary<T, Primary> primary;
198 static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
200 platform::store8( primary::ptr(pDest), primary::val(v), order );
202 static void atomic_store_explicit( T * pDest, T v, memory_order order ) CDS_NOEXCEPT
204 platform::store8( primary::ptr(pDest), primary::val(v), order );
206 static void atomic_store( T volatile * pDest, T v ) CDS_NOEXCEPT
208 atomic_store_explicit( pDest, v, memory_order_seq_cst );
210 static void atomic_store( T * pDest, T v ) CDS_NOEXCEPT
212 atomic_store_explicit( pDest, v, memory_order_seq_cst );
216 static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
218 return primary::ret( platform::load8( primary::ptr(pSrc), order ));
220 static T atomic_load_explicit( T const * pSrc, memory_order order ) CDS_NOEXCEPT
222 return primary::ret( platform::load8( primary::ptr(pSrc), order ));
224 static T atomic_load( T volatile const * pSrc ) CDS_NOEXCEPT
226 return atomic_load_explicit( pSrc, memory_order_seq_cst );
228 static T atomic_load( T const * pSrc ) CDS_NOEXCEPT
230 return atomic_load_explicit( pSrc, memory_order_seq_cst );
234 static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
236 return primary::ret( platform::exchange8( primary::ptr(pDest), primary::val(val), order ));
238 static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) CDS_NOEXCEPT
240 return primary::ret( platform::exchange8( primary::ptr(pDest), primary::val(val), order ));
242 static T atomic_exchange( T volatile * pDest, T val ) CDS_NOEXCEPT
244 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
246 static T atomic_exchange( T * pDest, T val ) CDS_NOEXCEPT
248 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
252 static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
255 return platform::cas8_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
257 static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
260 return platform::cas8_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
262 static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
264 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
266 static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
268 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
270 static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
273 return platform::cas8_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
275 static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
278 return platform::cas8_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
280 static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
282 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
284 static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
286 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
290 template <typename T>
291 struct atomic_integral_ops< T, 1 >
292 : atomic_generic_ops<T, 1, T >
293 , atomic_integral_bitwise_ops<T>
295 typedef atomic_integral_bitwise_ops<T> bitwise_ops;
298 static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
300 # ifdef CDS_ATOMIC_fetch8_add_defined
301 return platform::fetch8_add( pDest, val, order );
303 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
304 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
308 static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
310 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
312 static T atomic_fetch_add( T volatile * pDest, T val ) CDS_NOEXCEPT
314 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
316 static T atomic_fetch_add( T * pDest, T val ) CDS_NOEXCEPT
318 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
322 static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
324 # ifdef CDS_ATOMIC_fetch8_sub_defined
325 return platform::fetch8_sub( pDest, val, order );
327 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
328 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
332 static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
334 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
336 static T atomic_fetch_sub( T volatile * pDest, T val ) CDS_NOEXCEPT
338 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
340 static T atomic_fetch_sub( T * pDest, T val ) CDS_NOEXCEPT
342 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
346 static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
348 # ifdef CDS_ATOMIC_fetch8_and_defined
349 return platform::fetch8_and( pDest, val, order );
351 return bitwise_ops::fetch_and( pDest, val, order );
354 static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
356 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
358 static T atomic_fetch_and( T volatile * pDest, T val ) CDS_NOEXCEPT
360 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
362 static T atomic_fetch_and( T * pDest, T val ) CDS_NOEXCEPT
364 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
368 static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
370 # ifdef CDS_ATOMIC_fetch8_or_defined
371 return platform::fetch8_or( pDest, val, order );
373 return bitwise_ops::fetch_or( pDest, val, order );
376 static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
378 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
380 static T atomic_fetch_or( T volatile * pDest, T val ) CDS_NOEXCEPT
382 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
384 static T atomic_fetch_or( T * pDest, T val ) CDS_NOEXCEPT
386 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
390 static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
392 # ifdef CDS_ATOMIC_fetch8_xor_defined
393 return platform::fetch8_xor( pDest, val, order );
395 return bitwise_ops::fetch_xor( pDest, val, order );
398 static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
400 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
402 static T atomic_fetch_xor( T volatile * pDest, T val ) CDS_NOEXCEPT
404 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
406 static T atomic_fetch_xor( T * pDest, T val ) CDS_NOEXCEPT
408 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
412 // 16-bit atomic operations
414 template <typename T, typename Primary>
415 struct atomic_generic_ops< T, 2, Primary >
417 typedef make_atomic_primary<T, Primary> primary;
420 static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
422 platform::store16( primary::ptr(pDest), primary::val(v), order );
424 static void atomic_store_explicit( T * pDest, T v, memory_order order ) CDS_NOEXCEPT
426 platform::store16( primary::ptr(pDest), primary::val(v), order );
428 static void atomic_store( T volatile * pDest, T v ) CDS_NOEXCEPT
430 atomic_store_explicit( pDest, v, memory_order_seq_cst );
432 static void atomic_store( T * pDest, T v ) CDS_NOEXCEPT
434 atomic_store_explicit( pDest, v, memory_order_seq_cst );
438 static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
440 return primary::ret( platform::load16( primary::ptr(pSrc), order ));
442 static T atomic_load_explicit( T const * pSrc, memory_order order ) CDS_NOEXCEPT
444 return primary::ret( platform::load16( primary::ptr(pSrc), order ));
446 static T atomic_load( T volatile const * pSrc ) CDS_NOEXCEPT
448 return atomic_load_explicit( pSrc, memory_order_seq_cst );
450 static T atomic_load( T const * pSrc ) CDS_NOEXCEPT
452 return atomic_load_explicit( pSrc, memory_order_seq_cst );
456 static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
458 return primary::ret( platform::exchange16( primary::ptr(pDest), primary::val(val), order ));
460 static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) CDS_NOEXCEPT
462 return primary::ret( platform::exchange16( primary::ptr(pDest), primary::val(val), order ));
464 static T atomic_exchange( T volatile * pDest, T val ) CDS_NOEXCEPT
466 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
468 static T atomic_exchange( T * pDest, T val ) CDS_NOEXCEPT
470 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
474 static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
477 return platform::cas16_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
479 static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
482 return platform::cas16_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
484 static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
486 return atomic_compare_exchange_weak_explicit( pDest, expected, primary::val(desired), memory_order_seq_cst, memory_order_relaxed );
488 static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
490 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
492 static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
495 return platform::cas16_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
497 static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
500 return platform::cas16_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
502 static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
504 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
506 static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
508 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
512 template <typename T>
513 struct atomic_integral_ops< T, 2 >
514 : atomic_generic_ops< T, 2, T >
515 , atomic_integral_bitwise_ops<T>
517 typedef atomic_integral_bitwise_ops<T> bitwise_ops;
520 static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
522 # ifdef CDS_ATOMIC_fetch16_add_defined
523 return platform::fetch16_add( pDest, val, order );
525 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
526 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
530 static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
532 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
534 static T atomic_fetch_add( T volatile * pDest, T val ) CDS_NOEXCEPT
536 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
538 static T atomic_fetch_add( T * pDest, T val ) CDS_NOEXCEPT
540 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
544 static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
546 # ifdef CDS_ATOMIC_fetch16_sub_defined
547 return platform::fetch16_sub( pDest, val, order );
549 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
550 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
554 static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
556 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
558 static T atomic_fetch_sub( T volatile * pDest, T val ) CDS_NOEXCEPT
560 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
562 static T atomic_fetch_sub( T * pDest, T val ) CDS_NOEXCEPT
564 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
568 static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
570 # ifdef CDS_ATOMIC_fetch16_and_defined
571 return platform::fetch16_and( pDest, val, order );
573 return bitwise_ops::fetch_and( pDest, val, order );
576 static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
578 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
580 static T atomic_fetch_and( T volatile * pDest, T val ) CDS_NOEXCEPT
582 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
584 static T atomic_fetch_and( T * pDest, T val ) CDS_NOEXCEPT
586 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
590 static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
592 # ifdef CDS_ATOMIC_fetch16_or_defined
593 return platform::fetch16_or( pDest, val, order );
595 return bitwise_ops::fetch_or( pDest, val, order );
598 static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
600 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
602 static T atomic_fetch_or( T volatile * pDest, T val ) CDS_NOEXCEPT
604 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
606 static T atomic_fetch_or( T * pDest, T val ) CDS_NOEXCEPT
608 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
612 static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
614 # ifdef CDS_ATOMIC_fetch16_xor_defined
615 return platform::fetch16_xor( pDest, val, order );
617 return bitwise_ops::fetch_xor( pDest, val, order );
620 static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
622 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
624 static T atomic_fetch_xor( T volatile * pDest, T val ) CDS_NOEXCEPT
626 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
628 static T atomic_fetch_xor( T * pDest, T val ) CDS_NOEXCEPT
630 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
634 // 32-bit atomic operations
636 template <typename T, typename Primary>
637 struct atomic_generic_ops< T, 4, Primary >
639 typedef make_atomic_primary<T, Primary> primary;
642 static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
644 platform::store32( primary::ptr(pDest), primary::val(v), order );
646 static void atomic_store_explicit( T * pDest, T v, memory_order order ) CDS_NOEXCEPT
648 platform::store32( primary::ptr(pDest), primary::val(v), order );
650 static void atomic_store( T volatile * pDest, T v ) CDS_NOEXCEPT
652 atomic_store_explicit( pDest, v, memory_order_seq_cst );
654 static void atomic_store( T * pDest, T v ) CDS_NOEXCEPT
656 atomic_store_explicit( pDest, v, memory_order_seq_cst );
660 static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
662 return primary::ret( platform::load32( primary::ptr(pSrc), order ));
664 static T atomic_load_explicit( T const * pSrc, memory_order order ) CDS_NOEXCEPT
666 return primary::ret( platform::load32( primary::ptr(pSrc), order ));
668 static T atomic_load( T volatile const * pSrc ) CDS_NOEXCEPT
670 return atomic_load_explicit( pSrc, memory_order_seq_cst );
672 static T atomic_load( T const * pSrc ) CDS_NOEXCEPT
674 return atomic_load_explicit( pSrc, memory_order_seq_cst );
678 static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
680 return primary::ret( platform::exchange32( primary::ptr(pDest), primary::val(val), order ));
682 static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) CDS_NOEXCEPT
684 return primary::ret( platform::exchange32( primary::ptr(pDest), primary::val(val), order ));
686 static T atomic_exchange( T volatile * pDest, T val ) CDS_NOEXCEPT
688 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
690 static T atomic_exchange( T * pDest, T val ) CDS_NOEXCEPT
692 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
696 static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
699 return platform::cas32_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
701 static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
704 return platform::cas32_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
706 static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
708 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
710 static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
712 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
714 static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
717 return platform::cas32_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
719 static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
722 return platform::cas32_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
724 static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
726 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
728 static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
730 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
734 template <typename T>
735 struct atomic_integral_ops< T, 4 >
736 : atomic_generic_ops< T, 4, T >
737 , atomic_integral_bitwise_ops<T>
739 typedef atomic_integral_bitwise_ops<T> bitwise_ops;
741 static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
743 # ifdef CDS_ATOMIC_fetch32_add_defined
744 return platform::fetch32_add( pDest, val, order );
746 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
747 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
751 static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
753 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
755 static T atomic_fetch_add( T volatile * pDest, T val ) CDS_NOEXCEPT
757 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
759 static T atomic_fetch_add( T * pDest, T val ) CDS_NOEXCEPT
761 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
765 static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
767 # ifdef CDS_ATOMIC_fetch32_sub_defined
768 return platform::fetch32_sub( pDest, val, order );
770 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
771 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
775 static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
777 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
779 static T atomic_fetch_sub( T volatile * pDest, T val ) CDS_NOEXCEPT
781 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
783 static T atomic_fetch_sub( T * pDest, T val ) CDS_NOEXCEPT
785 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
789 static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
791 # ifdef CDS_ATOMIC_fetch32_and_defined
792 return platform::fetch32_and( pDest, val, order );
794 return bitwise_ops::fetch_and( pDest, val, order );
797 static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
799 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
801 static T atomic_fetch_and( T volatile * pDest, T val ) CDS_NOEXCEPT
803 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
805 static T atomic_fetch_and( T * pDest, T val ) CDS_NOEXCEPT
807 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
811 static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
813 # ifdef CDS_ATOMIC_fetch32_or_defined
814 return platform::fetch32_or( pDest, val, order );
816 return bitwise_ops::fetch_or( pDest, val, order );
819 static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
821 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
823 static T atomic_fetch_or( T volatile * pDest, T val ) CDS_NOEXCEPT
825 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
827 static T atomic_fetch_or( T * pDest, T val ) CDS_NOEXCEPT
829 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
833 static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
835 # ifdef CDS_ATOMIC_fetch32_xor_defined
836 return platform::fetch32_xor( pDest, val, order );
838 return bitwise_ops::fetch_xor( pDest, val, order );
841 static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
843 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
845 static T atomic_fetch_xor( T volatile * pDest, T val ) CDS_NOEXCEPT
847 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
849 static T atomic_fetch_xor( T * pDest, T val ) CDS_NOEXCEPT
851 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
856 // 64-bit atomic operations
858 template <typename T, typename Primary>
859 struct atomic_generic_ops< T, 8, Primary >
861 typedef make_atomic_primary<T, Primary> primary;
864 static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
866 platform::store64( primary::ptr(pDest), primary::val(v), order );
868 static void atomic_store_explicit( T * pDest, T v, memory_order order ) CDS_NOEXCEPT
870 platform::store64( primary::ptr(pDest), primary::val(v), order );
872 static void atomic_store( T volatile * pDest, T v ) CDS_NOEXCEPT
874 atomic_store_explicit( pDest, v, memory_order_seq_cst );
876 static void atomic_store( T * pDest, T v ) CDS_NOEXCEPT
878 atomic_store_explicit( pDest, v, memory_order_seq_cst );
882 static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
884 return primary::ret( platform::load64( primary::ptr(pSrc), order ));
886 static T atomic_load_explicit( T const * pSrc, memory_order order ) CDS_NOEXCEPT
888 return primary::ret( platform::load64( primary::ptr(pSrc), order ));
890 static T atomic_load( T volatile const * pSrc ) CDS_NOEXCEPT
892 return atomic_load_explicit( pSrc, memory_order_seq_cst );
894 static T atomic_load( T const * pSrc ) CDS_NOEXCEPT
896 return atomic_load_explicit( pSrc, memory_order_seq_cst );
900 static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
902 return primary::ret( platform::exchange64( primary::ptr(pDest), primary::val(val), order ));
904 static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) CDS_NOEXCEPT
906 return primary::ret( platform::exchange64( primary::ptr(pDest), primary::val(val), order ));
908 static T atomic_exchange( T volatile * pDest, T val ) CDS_NOEXCEPT
910 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
912 static T atomic_exchange( T * pDest, T val ) CDS_NOEXCEPT
914 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
918 static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
921 return platform::cas64_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
923 static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
926 return platform::cas64_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
928 static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
930 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
932 static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
934 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
936 static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
939 return platform::cas64_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
941 static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
944 return platform::cas64_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
946 static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
948 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
950 static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
952 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
957 template <typename T>
958 struct atomic_integral_ops< T, 8 >
959 : atomic_generic_ops< T, 8, T >
960 , atomic_integral_bitwise_ops<T>
962 typedef atomic_integral_bitwise_ops<T> bitwise_ops;
963 typedef atomic_generic_ops<T, 8, T> general_ops;
966 static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
968 # ifdef CDS_ATOMIC_fetch64_add_defined
969 return platform::fetch64_add( pDest, val, order );
971 T cur = general_ops::atomic_load_explicit( pDest, memory_order_relaxed );
972 do {} while ( !general_ops::atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
976 static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
978 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
980 static T atomic_fetch_add( T volatile * pDest, T val ) CDS_NOEXCEPT
982 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
984 static T atomic_fetch_add( T * pDest, T val ) CDS_NOEXCEPT
986 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
990 static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
992 # ifdef CDS_ATOMIC_fetch64_sub_defined
993 return platform::fetch64_sub( pDest, val, order );
995 T cur = general_ops::atomic_load_explicit( pDest, memory_order_relaxed );
996 do {} while ( !general_ops::atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
1000 static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
1002 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1004 static T atomic_fetch_sub( T volatile * pDest, T val ) CDS_NOEXCEPT
1006 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1008 static T atomic_fetch_sub( T * pDest, T val ) CDS_NOEXCEPT
1010 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1014 static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
1016 # ifdef CDS_ATOMIC_fetch64_and_defined
1017 return platform::fetch64_and( pDest, val, order );
1019 return bitwise_ops::fetch_and( pDest, val, order );
1022 static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
1024 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1026 static T atomic_fetch_and( T volatile * pDest, T val ) CDS_NOEXCEPT
1028 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
1030 static T atomic_fetch_and( T * pDest, T val ) CDS_NOEXCEPT
1032 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
1036 static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
1038 # ifdef CDS_ATOMIC_fetch64_or_defined
1039 return platform::fetch64_or( pDest, val, order );
1041 return bitwise_ops::fetch_or( pDest, val, order );
1044 static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
1046 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1048 static T atomic_fetch_or( T volatile * pDest, T val ) CDS_NOEXCEPT
1050 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
1052 static T atomic_fetch_or( T * pDest, T val ) CDS_NOEXCEPT
1054 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
1058 static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
1060 # ifdef CDS_ATOMIC_fetch64_xor_defined
1061 return platform::fetch64_xor( pDest, val, order );
1063 return bitwise_ops::fetch_xor( pDest, val, order );
1066 static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
1068 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1070 static T atomic_fetch_xor( T volatile * pDest, T val ) CDS_NOEXCEPT
1072 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
1074 static T atomic_fetch_xor( T * pDest, T val ) CDS_NOEXCEPT
1076 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
1081 // atomic pointer operations
1082 template <typename T>
1083 struct atomic_pointer_base
1086 static void atomic_store_explicit( T * volatile * pDest, T * v, memory_order order ) CDS_NOEXCEPT
1088 platform::store_ptr( pDest, v, order );
1090 static void atomic_store_explicit( T * * pDest, T * v, memory_order order ) CDS_NOEXCEPT
1092 platform::store_ptr( pDest, v, order );
1094 static void atomic_store( T * volatile * pDest, T * v ) CDS_NOEXCEPT
1096 atomic_store_explicit( pDest, v, memory_order_seq_cst );
1098 static void atomic_store( T * * pDest, T * v ) CDS_NOEXCEPT
1100 atomic_store_explicit( pDest, v, memory_order_seq_cst );
1104 static T * atomic_load_explicit( T * volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
1106 return platform::load_ptr( pSrc, order );
1108 static T * atomic_load_explicit( T * const * pSrc, memory_order order ) CDS_NOEXCEPT
1110 return platform::load_ptr( pSrc, order );
1112 static T * atomic_load( T * volatile const * pSrc ) CDS_NOEXCEPT
1114 return atomic_load_explicit( pSrc, memory_order_seq_cst );
1116 static T * atomic_load( T * const * pSrc ) CDS_NOEXCEPT
1118 return atomic_load_explicit( pSrc, memory_order_seq_cst );
1122 static T * atomic_exchange_explicit( T * volatile * pDest, T * val, memory_order order ) CDS_NOEXCEPT
1124 return platform::exchange_ptr( pDest, val, order );
1126 static T * atomic_exchange_explicit( T * * pDest, T * val, memory_order order ) CDS_NOEXCEPT
1128 return platform::exchange_ptr( pDest, val, order );
1130 static T * atomic_exchange( T * volatile * pDest, T * val ) CDS_NOEXCEPT
1132 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
1134 static T * atomic_exchange( T * * pDest, T * val ) CDS_NOEXCEPT
1136 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
1140 static bool atomic_compare_exchange_weak_explicit( T * volatile * pDest, T * * expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
1143 return platform::cas_ptr_weak( pDest, *expected, desired, mo_success, mo_fail );
1145 static bool atomic_compare_exchange_weak_explicit( T * * pDest, T * * expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
1148 return platform::cas_ptr_weak( pDest, *expected, desired, mo_success, mo_fail );
1150 static bool atomic_compare_exchange_weak( T * volatile * pDest, T ** expected, T * desired ) CDS_NOEXCEPT
1152 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1154 static bool atomic_compare_exchange_weak( T ** pDest, T ** expected, T * desired ) CDS_NOEXCEPT
1156 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1158 static bool atomic_compare_exchange_strong_explicit( T * volatile * pDest, T ** expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
1161 return platform::cas_ptr_strong( pDest, *expected, desired, mo_success, mo_fail );
1163 static bool atomic_compare_exchange_strong_explicit( T ** pDest, T ** expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
1166 return platform::cas_ptr_strong( pDest, *expected, desired, mo_success, mo_fail );
1168 static bool atomic_compare_exchange_strong( T * volatile * pDest, T ** expected, T * desired ) CDS_NOEXCEPT
1170 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1172 static bool atomic_compare_exchange_strong( T ** pDest, T ** expected, T * desired ) CDS_NOEXCEPT
1174 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1178 template <typename T>
1179 struct atomic_pointer: public atomic_pointer_base<T>
1181 typedef atomic_pointer_base<T> base_class;
1183 static T * atomic_fetch_add_explicit(T * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
1185 # ifdef CDS_ATOMIC_fetch_ptr_add_defined
1186 platform::fetch_ptr_add( pDest, val, order );
1188 T * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1189 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
1193 static T * atomic_fetch_add_explicit(T * * pDest, ptrdiff_t val , memory_order order) CDS_NOEXCEPT
1195 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1197 static T * atomic_fetch_add( T * volatile * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1199 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1201 static T * atomic_fetch_add( T ** pDest, ptrdiff_t val ) CDS_NOEXCEPT
1203 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1207 static T * atomic_fetch_sub_explicit(T * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
1209 # ifdef CDS_ATOMIC_fetch_ptr_sub_defined
1210 platform::fetch_ptr_sub( pDest, val, order );
1212 T * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1213 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
1217 static T * atomic_fetch_sub_explicit(T ** pDest, ptrdiff_t val , memory_order order) CDS_NOEXCEPT
1219 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1221 static T * atomic_fetch_sub( T volatile * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1223 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1225 static T * atomic_fetch_sub( T * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1227 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1232 struct atomic_pointer<void>: public atomic_pointer_base<void>
1234 typedef atomic_pointer_base<void> base_class;
1237 static void * atomic_fetch_add_explicit(void * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
1239 void * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1240 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, reinterpret_cast<char *>(cur) + val, order, memory_order_relaxed ));
1243 static void * atomic_fetch_add_explicit(void * * pDest, ptrdiff_t val , memory_order order) CDS_NOEXCEPT
1245 return atomic_fetch_add_explicit( reinterpret_cast<void * volatile *>( pDest ), val, order );
1247 static void * atomic_fetch_add( void * volatile * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1249 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1251 static void * atomic_fetch_add( void ** pDest, ptrdiff_t val ) CDS_NOEXCEPT
1253 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1257 static void * atomic_fetch_sub_explicit(void * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
1259 void * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1260 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, reinterpret_cast<char *>(cur) - val, order, memory_order_relaxed ));
1263 static void * atomic_fetch_sub_explicit(void ** pDest, ptrdiff_t val , memory_order order) CDS_NOEXCEPT
1265 return atomic_fetch_sub_explicit( reinterpret_cast<void * volatile *>( pDest ), val, order );
1267 static void * atomic_fetch_sub( void * volatile * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1269 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1271 static void * atomic_fetch_sub( void ** pDest, ptrdiff_t val ) CDS_NOEXCEPT
1273 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1277 template <typename T>
1278 struct atomic_integral
1281 typename cds::details::aligned_type<T, sizeof(T)>::type volatile m_val;
1283 typedef atomic_integral_ops<T, sizeof(T)> atomic_ops;
1285 typedef T atomic_type;
1287 bool is_lock_free() const volatile CDS_NOEXCEPT
1291 bool is_lock_free() const CDS_NOEXCEPT
1295 void store(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1297 atomic_ops::atomic_store_explicit( &m_val, val, order );
1299 void store(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1301 atomic_ops::atomic_store_explicit( &m_val, val, order );
1304 T load(memory_order order = memory_order_seq_cst) const volatile CDS_NOEXCEPT
1306 return atomic_ops::atomic_load_explicit( &m_val, order );
1308 T load(memory_order order = memory_order_seq_cst) const CDS_NOEXCEPT
1310 return atomic_ops::atomic_load_explicit( &m_val, order );
1313 operator T() const volatile CDS_NOEXCEPT
1317 operator T() const CDS_NOEXCEPT
1322 T exchange(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1324 return atomic_ops::atomic_exchange_explicit( &m_val, val, order );
1326 T exchange(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1328 return atomic_ops::atomic_exchange_explicit( &m_val, val, order );
1331 bool compare_exchange_weak(T& expected, T desired , memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1333 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_val, &expected, desired, success_order, failure_order );
1335 bool compare_exchange_weak(T& expected, T desired , memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1337 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_val, &expected, desired, success_order, failure_order );
1339 bool compare_exchange_strong(T& expected, T desired , memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1341 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_val, &expected, desired, success_order, failure_order );
1343 bool compare_exchange_strong(T& expected, T desired , memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1345 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_val, &expected, desired, success_order, failure_order );
1347 bool compare_exchange_weak(T& expected, T desired , memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1349 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1351 bool compare_exchange_weak(T& expected, T desired , memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1353 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1355 bool compare_exchange_strong(T& expected, T desired , memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1357 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1359 bool compare_exchange_strong(T& expected, T desired , memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1361 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1364 T fetch_add(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1366 return atomic_ops::atomic_fetch_add_explicit( &m_val, val, order );
1368 T fetch_add(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1370 return atomic_ops::atomic_fetch_add_explicit( &m_val, val, order );
1372 T fetch_sub(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1374 return atomic_ops::atomic_fetch_sub_explicit( &m_val, val, order );
1376 T fetch_sub(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1378 return atomic_ops::atomic_fetch_sub_explicit( &m_val, val, order );
1380 T fetch_and(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1382 return atomic_ops::atomic_fetch_and_explicit( &m_val, val, order );
1384 T fetch_and(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1386 return atomic_ops::atomic_fetch_and_explicit( &m_val, val, order );
1389 T fetch_or(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1391 return atomic_ops::atomic_fetch_or_explicit( &m_val, val, order );
1393 T fetch_or(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1395 return atomic_ops::atomic_fetch_or_explicit( &m_val, val, order );
1397 T fetch_xor(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1399 return atomic_ops::atomic_fetch_xor_explicit( &m_val, val, order );
1401 T fetch_xor(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1403 return atomic_ops::atomic_fetch_xor_explicit( &m_val, val, order );
1406 atomic_integral() = default;
1407 CDS_CONSTEXPR atomic_integral(T val) CDS_NOEXCEPT
1411 atomic_integral(const atomic_integral&) = delete;
1412 atomic_integral& operator=(const atomic_integral&) = delete;
1413 atomic_integral& operator=(const atomic_integral&) volatile = delete;
1415 T operator=(T val) volatile CDS_NOEXCEPT
1420 T operator=(T val) CDS_NOEXCEPT
1427 T operator++(int) volatile CDS_NOEXCEPT
1429 return fetch_add( 1 );
1431 T operator++(int) CDS_NOEXCEPT
1433 return fetch_add( 1 );
1435 T operator--(int) volatile CDS_NOEXCEPT
1437 return fetch_sub( 1 );
1439 T operator--(int) CDS_NOEXCEPT
1441 return fetch_sub( 1 );
1445 T operator++() volatile CDS_NOEXCEPT
1447 return fetch_add( 1 ) + 1;
1449 T operator++() CDS_NOEXCEPT
1451 return fetch_add( 1 ) + 1;
1453 T operator--() volatile CDS_NOEXCEPT
1455 return fetch_sub( 1 ) - 1;
1457 T operator--() CDS_NOEXCEPT
1459 return fetch_sub( 1 ) - 1;
1463 T operator+=(T val) volatile CDS_NOEXCEPT
1465 return fetch_add( val ) + val;
1467 T operator+=(T val) CDS_NOEXCEPT
1469 return fetch_add( val ) + val;
1471 T operator-=(T val) volatile CDS_NOEXCEPT
1473 return fetch_sub( val ) - val;
1475 T operator-=(T val) CDS_NOEXCEPT
1477 return fetch_sub( val ) - val;
1479 T operator&=(T val) volatile CDS_NOEXCEPT
1481 return fetch_and( val ) & val;
1483 T operator&=(T val) CDS_NOEXCEPT
1485 return fetch_and( val ) & val;
1487 T operator|=(T val) volatile CDS_NOEXCEPT
1489 return fetch_or( val ) | val;
1491 T operator|=(T val) CDS_NOEXCEPT
1493 return fetch_or( val ) | val;
1495 T operator^=(T val) volatile CDS_NOEXCEPT
1497 return fetch_xor( val ) ^ val;
1499 T operator^=(T val) CDS_NOEXCEPT
1501 return fetch_xor( val ) ^ val;
1505 template <typename Type>
1506 struct select_primary_type {
1507 typedef typename details::primary_type<sizeof(Type)>::type type;
1510 struct select_primary_type<bool> {
1514 } // namespace details
1520 typedef details::atomic_generic_ops<T, sizeof(T), typename details::select_primary_type<T>::type > atomic_ops;
1524 bool is_lock_free() const volatile CDS_NOEXCEPT
1528 bool is_lock_free() const CDS_NOEXCEPT
1533 void store(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1535 atomic_ops::atomic_store_explicit( &m_data, val, order );
1537 void store(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1539 atomic_ops::atomic_store_explicit( &m_data, val, order );
1542 T load(memory_order order = memory_order_seq_cst) const volatile CDS_NOEXCEPT
1544 return atomic_ops::atomic_load_explicit( &m_data, order );
1546 T load(memory_order order = memory_order_seq_cst) const CDS_NOEXCEPT
1548 return atomic_ops::atomic_load_explicit( &m_data, order );
1551 operator T() const volatile CDS_NOEXCEPT
1555 operator T() const CDS_NOEXCEPT
1560 T exchange(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1562 return atomic_ops::atomic_exchange_explicit( &m_data, val, order );
1564 T exchange(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1566 return atomic_ops::atomic_exchange_explicit( &m_data, val, order );
1569 bool compare_exchange_weak(T& expected, T desired, memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1571 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_data, &expected, desired, success_order, failure_order );
1573 bool compare_exchange_weak(T& expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1575 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_data, &expected, desired, success_order, failure_order );
1577 bool compare_exchange_strong(T& expected, T desired, memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1579 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_data, &expected, desired, success_order, failure_order );
1581 bool compare_exchange_strong(T& expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1583 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_data, &expected, desired, success_order, failure_order );
1585 bool compare_exchange_weak(T& expected, T desired, memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1587 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1589 bool compare_exchange_weak(T& expected, T desired, memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1591 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1593 bool compare_exchange_strong(T& expected, T desired, memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1595 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1597 bool compare_exchange_strong(T& expected, T desired, memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1599 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1603 CDS_CONSTEXPR atomic(T val)
1607 atomic(const atomic&) = delete;
1608 atomic& operator=(const atomic&) = delete;
1609 atomic& operator=(const atomic&) volatile = delete;
1611 T operator=(T val) volatile CDS_NOEXCEPT
1616 T operator=(T val) CDS_NOEXCEPT
1623 # define CDS_DECLARE_ATOMIC_INTEGRAL( _type ) \
1625 struct atomic<_type>: public details::atomic_integral<_type> \
1628 typedef details::atomic_integral<_type> base_class ; \
1630 atomic() = default; \
1631 atomic(_type val) CDS_NOEXCEPT : base_class(val) {} \
1632 atomic(const atomic&) = delete; \
1633 atomic& operator=(const atomic&) = delete; \
1634 atomic& operator=(const atomic&) volatile = delete; \
1635 _type operator=(_type val) volatile CDS_NOEXCEPT { return base_class::operator=(val); } \
1636 _type operator=(_type val) CDS_NOEXCEPT { return base_class::operator=(val); } \
1639 CDS_DECLARE_ATOMIC_INTEGRAL(char)
1640 CDS_DECLARE_ATOMIC_INTEGRAL(signed char)
1641 CDS_DECLARE_ATOMIC_INTEGRAL(unsigned char)
1642 CDS_DECLARE_ATOMIC_INTEGRAL(short)
1643 CDS_DECLARE_ATOMIC_INTEGRAL(unsigned short)
1644 CDS_DECLARE_ATOMIC_INTEGRAL(int)
1645 CDS_DECLARE_ATOMIC_INTEGRAL(unsigned int)
1646 CDS_DECLARE_ATOMIC_INTEGRAL(long)
1647 CDS_DECLARE_ATOMIC_INTEGRAL(unsigned long)
1648 CDS_DECLARE_ATOMIC_INTEGRAL(long long)
1649 CDS_DECLARE_ATOMIC_INTEGRAL(unsigned long long)
1650 //#if CDS_COMPILER == CDS_COMPILER_GCC && CDS_COMPILER_VERSION >= 40400
1651 // CDS_DECLARE_ATOMIC_INTEGRAL(char16_t)
1652 // CDS_DECLARE_ATOMIC_INTEGRAL(char32_t)
1654 // CDS_DECLARE_ATOMIC_INTEGRAL(wchar_t)
1656 # undef CDS_DECLARE_ATOMIC_INTEGRAL
1659 template <typename T>
1664 typedef details::atomic_pointer<T> atomic_ops;
1666 bool is_lock_free() const volatile CDS_NOEXCEPT
1670 bool is_lock_free() const CDS_NOEXCEPT
1675 void store(T * val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1677 atomic_ops::atomic_store_explicit( &m_ptr, val, order );
1679 void store(T * val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1681 atomic_ops::atomic_store_explicit( &m_ptr, val, order );
1684 T * load(memory_order order = memory_order_seq_cst) const volatile CDS_NOEXCEPT
1686 return atomic_ops::atomic_load_explicit( &m_ptr, order );
1688 T * load(memory_order order = memory_order_seq_cst) const CDS_NOEXCEPT
1690 return atomic_ops::atomic_load_explicit( &m_ptr, order );
1693 operator T *() const volatile CDS_NOEXCEPT
1697 operator T *() const CDS_NOEXCEPT
1702 T * exchange(T * val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1704 return atomic_ops::atomic_exchange_explicit( &m_ptr, val, order );
1706 T * exchange(T * val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1708 return atomic_ops::atomic_exchange_explicit( &m_ptr, val, order );
1711 bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1713 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1715 bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1717 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1719 bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1721 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1723 bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1725 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1727 bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1729 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1731 bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1733 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1735 bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1737 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1739 bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1741 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1744 T * fetch_add(ptrdiff_t offset, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1746 return atomic_ops::atomic_fetch_add_explicit( &m_ptr, offset, order );
1748 T * fetch_add(ptrdiff_t offset, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1750 return atomic_ops::atomic_fetch_add_explicit( &m_ptr, offset, order );
1753 T * fetch_sub(ptrdiff_t offset, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1755 return atomic_ops::atomic_fetch_sub_explicit( &m_ptr, offset, order );
1757 T * fetch_sub(ptrdiff_t offset, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1759 return atomic_ops::atomic_fetch_sub_explicit( &m_ptr, offset, order );
1763 CDS_CONSTEXPR atomic(T * val) CDS_NOEXCEPT
1767 atomic(const atomic&) = delete;
1768 atomic& operator=(const atomic&) = delete;
1769 atomic& operator=(const atomic&) volatile = delete;
1771 T * operator=(T * val) volatile CDS_NOEXCEPT
1776 T * operator=(T * val) CDS_NOEXCEPT
1784 typedef atomic<bool> atomic_bool;
1785 typedef atomic<char> atomic_char;
1786 typedef atomic<signed char> atomic_schar;
1787 typedef atomic<unsigned char> atomic_uchar;
1788 typedef atomic<short> atomic_short;
1789 typedef atomic<unsigned short> atomic_ushort;
1790 typedef atomic<int> atomic_int;
1791 typedef atomic<unsigned int> atomic_uint;
1792 typedef atomic<long> atomic_long;
1793 typedef atomic<unsigned long> atomic_ulong;
1794 typedef atomic<long long> atomic_llong;
1795 typedef atomic<unsigned long long> atomic_ullong;
1796 #if ( CDS_COMPILER == CDS_COMPILER_GCC && CDS_COMPILER_VERSION >= 40400 ) || CDS_COMPILER == CDS_COMPILER_CLANG
1797 typedef atomic<char16_t> atomic_char16_t;
1798 typedef atomic<char32_t> atomic_char32_t;
1800 typedef atomic<wchar_t> atomic_wchar_t;
1803 typedef atomic<std::int_least8_t> atomic_int_least8_t;
1804 typedef atomic<std::uint_least8_t> atomic_uint_least8_t;
1805 typedef atomic<std::int_least16_t> atomic_int_least16_t;
1806 typedef atomic<std::uint_least16_t> atomic_uint_least16_t;
1807 typedef atomic<std::int_least32_t> atomic_int_least32_t;
1808 typedef atomic<std::uint_least32_t> atomic_uint_least32_t;
1809 typedef atomic<std::int_least64_t> atomic_int_least64_t;
1810 typedef atomic<std::uint_least64_t> atomic_uint_least64_t;
1811 typedef atomic<std::int_fast8_t> atomic_int_fast8_t;
1812 typedef atomic<std::uint_fast8_t> atomic_uint_fast8_t;
1813 typedef atomic<std::int_fast16_t> atomic_int_fast16_t;
1814 typedef atomic<std::uint_fast16_t> atomic_uint_fast16_t;
1815 typedef atomic<std::int_fast32_t> atomic_int_fast32_t;
1816 typedef atomic<std::uint_fast32_t> atomic_uint_fast32_t;
1817 typedef atomic<std::int_fast64_t> atomic_int_fast64_t;
1818 typedef atomic<std::uint_fast64_t> atomic_uint_fast64_t;
1819 typedef atomic<intptr_t> atomic_intptr_t;
1820 typedef atomic<uintptr_t> atomic_uintptr_t;
1821 typedef atomic<size_t> atomic_size_t;
1822 typedef atomic<ptrdiff_t> atomic_ptrdiff_t;
1823 typedef atomic<std::intmax_t> atomic_intmax_t;
1824 typedef atomic<std::uintmax_t> atomic_uintmax_t;
1827 static inline bool atomic_is_lock_free(const volatile atomic<T> * p) CDS_NOEXCEPT
1829 return p->is_lock_free();
1833 static inline bool atomic_is_lock_free(const atomic<T> * p ) CDS_NOEXCEPT
1835 return p->is_lock_free();
1840 static inline void atomic_init(volatile atomic<T> * p, T val) CDS_NOEXCEPT
1846 static inline void atomic_init( atomic<T> * p, T val) CDS_NOEXCEPT
1853 static inline void atomic_store(volatile atomic<T>* p, T val) CDS_NOEXCEPT
1858 static inline void atomic_store(atomic<T>* p, T val) CDS_NOEXCEPT
1864 static inline void atomic_store_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1866 p->store( val, order );
1869 static inline void atomic_store_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1871 p->store( val, order );
1875 static inline T atomic_load(const volatile atomic<T>* p) CDS_NOEXCEPT
1880 static inline T atomic_load(const atomic<T>* p) CDS_NOEXCEPT
1886 static inline T atomic_load_explicit(const volatile atomic<T>* p, memory_order order) CDS_NOEXCEPT
1888 return p->load( order );
1891 static inline T atomic_load_explicit(const atomic<T>* p, memory_order order) CDS_NOEXCEPT
1893 return p->load( order );
1897 static inline T atomic_exchange(volatile atomic<T>* p, T val) CDS_NOEXCEPT
1899 return p->exchange( val );
1902 static inline T atomic_exchange(atomic<T>* p, T val ) CDS_NOEXCEPT
1904 return p->exchange( val );
1908 static inline T atomic_exchange_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1910 return p->exchange( val, order );
1913 static inline T atomic_exchange_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1915 return p->exchange( val, order );
1919 static inline bool atomic_compare_exchange_weak(volatile atomic<T>* p, T* expected, T desired) CDS_NOEXCEPT
1921 return p->compare_exchange_weak( *expected, desired );
1924 static inline bool atomic_compare_exchange_weak(atomic<T>* p, T* expected, T desired) CDS_NOEXCEPT
1926 return p->compare_exchange_weak( *expected, desired );
1930 static inline bool atomic_compare_exchange_strong(volatile atomic<T>* p, T* expected, T desired) CDS_NOEXCEPT
1932 return p->compare_exchange_strong( *expected, desired );
1935 static inline bool atomic_compare_exchange_strong(atomic<T>* p, T* expected, T desired) CDS_NOEXCEPT
1937 return p->compare_exchange_strong( *expected, desired );
1941 static inline bool atomic_compare_exchange_weak_explicit(volatile atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1943 return p->compare_exchange_weak( *expected, desired, success_order, failure_order );
1946 static inline bool atomic_compare_exchange_weak_explicit(atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1948 return p->compare_exchange_weak( *expected, desired, success_order, failure_order );
1952 static inline bool atomic_compare_exchange_strong_explicit(volatile atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1954 return p->compare_exchange_strong( *expected, desired, success_order, failure_order );
1957 static inline bool atomic_compare_exchange_strong_explicit(atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1959 return p->compare_exchange_strong( *expected, desired, success_order, failure_order );
1963 static inline T atomic_fetch_add(volatile atomic<T>* p, T val) CDS_NOEXCEPT
1965 return p->fetch_add( val );
1968 static inline T atomic_fetch_add(atomic<T>* p, T val) CDS_NOEXCEPT
1970 return p->fetch_add( val );
1973 static inline T * atomic_fetch_add(volatile atomic<T *>* p, ptrdiff_t offset) CDS_NOEXCEPT
1975 return p->fetch_add( offset );
1978 static inline T * atomic_fetch_add(atomic<T *>* p, ptrdiff_t offset) CDS_NOEXCEPT
1980 return p->fetch_add( offset );
1984 static inline T atomic_fetch_add_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1986 return p->fetch_add( val, order );
1989 static inline T atomic_fetch_add_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1991 return p->fetch_add( val, order );
1994 static inline T * atomic_fetch_add_explicit(volatile atomic<T *>* p, ptrdiff_t offset, memory_order order) CDS_NOEXCEPT
1996 return p->fetch_add( offset, order );
1999 static inline T * atomic_fetch_add_explicit(atomic<T *>* p, ptrdiff_t offset, memory_order order) CDS_NOEXCEPT
2001 return p->fetch_add( offset, order );
2005 static inline T atomic_fetch_sub(volatile atomic<T>* p, T val) CDS_NOEXCEPT
2007 return p->fetch_sub( val );
2010 static inline T atomic_fetch_sub(atomic<T>* p, T val) CDS_NOEXCEPT
2012 return p->fetch_sub( val );
2015 static inline T * atomic_fetch_sub(volatile atomic<T *>* p, ptrdiff_t offset) CDS_NOEXCEPT
2017 return p->fetch_sub( offset );
2020 static inline T * atomic_fetch_sub(atomic<T *>* p, ptrdiff_t offset) CDS_NOEXCEPT
2022 return p->fetch_sub( offset );
2026 static inline T atomic_fetch_sub_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2028 return p->fetch_sub( val, order );
2031 static inline T atomic_fetch_sub_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2033 return p->fetch_sub( val, order );
2036 static inline T * atomic_fetch_sub_explicit(volatile atomic<T *>* p, ptrdiff_t offset, memory_order order) CDS_NOEXCEPT
2038 return p->fetch_sub( offset, order );
2041 static inline T * atomic_fetch_sub_explicit(atomic<T *>* p, ptrdiff_t offset, memory_order order) CDS_NOEXCEPT
2043 return p->fetch_sub( offset, order );
2047 static inline T atomic_fetch_and(volatile atomic<T>* p, T val) CDS_NOEXCEPT
2049 return p->fetch_and( val );
2052 static inline T atomic_fetch_and(atomic<T>* p, T val) CDS_NOEXCEPT
2054 return p->fetch_and( val );
2058 static inline T atomic_fetch_and_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2060 return p->fetch_and( val, order );
2063 static inline T atomic_fetch_and_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2065 return p->fetch_and( val, order );
2069 static inline T atomic_fetch_or(volatile atomic<T>* p, T val) CDS_NOEXCEPT
2071 return p->fetch_or( val );
2074 static inline T atomic_fetch_or(atomic<T>* p, T val) CDS_NOEXCEPT
2076 return p->fetch_or( val );
2080 static inline T atomic_fetch_or_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2082 return p->fetch_or( val, order );
2085 static inline T atomic_fetch_or_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2087 return p->fetch_or( val, order );
2091 static inline T atomic_fetch_xor(volatile atomic<T>* p, T val) CDS_NOEXCEPT
2093 return p->fetch_xor( val );
2096 static inline T atomic_fetch_xor(atomic<T>* p, T val) CDS_NOEXCEPT
2098 return p->fetch_xor( val );
2102 static inline T atomic_fetch_xor_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2104 return p->fetch_xor( val, order );
2107 static inline T atomic_fetch_xor_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2109 return p->fetch_xor( val, order );
2113 typedef struct atomic_flag
2115 void clear( memory_order order = memory_order_seq_cst ) volatile CDS_NOEXCEPT
2117 assert( order != memory_order_acquire
2118 && order != memory_order_acq_rel
2119 && order != memory_order_consume
2121 platform::atomic_flag_clear( &m_Flag, order );
2123 void clear( memory_order order = memory_order_seq_cst ) CDS_NOEXCEPT
2125 assert( order != memory_order_acquire
2126 && order != memory_order_acq_rel
2127 && order != memory_order_consume
2129 platform::atomic_flag_clear( &m_Flag, order );
2132 bool test_and_set( memory_order order = memory_order_seq_cst ) volatile CDS_NOEXCEPT
2134 return platform::atomic_flag_tas( &m_Flag, order );
2136 bool test_and_set( memory_order order = memory_order_seq_cst ) CDS_NOEXCEPT
2138 return platform::atomic_flag_tas( &m_Flag, order );
2141 atomic_flag() = default;
2143 atomic_flag(const atomic_flag&) = delete;
2144 atomic_flag& operator=(const atomic_flag&) = delete;
2145 atomic_flag& operator=(const atomic_flag&) volatile = delete;
2147 platform::atomic_flag_type volatile m_Flag;
2150 static inline bool atomic_flag_test_and_set(volatile atomic_flag* p) CDS_NOEXCEPT
2152 return p->test_and_set();
2154 static inline bool atomic_flag_test_and_set(atomic_flag * p) CDS_NOEXCEPT
2156 return p->test_and_set();
2158 static inline bool atomic_flag_test_and_set_explicit(volatile atomic_flag* p, memory_order order) CDS_NOEXCEPT
2160 return p->test_and_set( order );
2162 static inline bool atomic_flag_test_and_set_explicit(atomic_flag* p, memory_order order) CDS_NOEXCEPT
2164 return p->test_and_set( order );
2166 static inline void atomic_flag_clear(volatile atomic_flag* p) CDS_NOEXCEPT
2170 static inline void atomic_flag_clear(atomic_flag* p) CDS_NOEXCEPT
2174 static inline void atomic_flag_clear_explicit(volatile atomic_flag* p, memory_order order) CDS_NOEXCEPT
2176 return p->clear( order );
2178 static inline void atomic_flag_clear_explicit(atomic_flag* p, memory_order order) CDS_NOEXCEPT
2180 return p->clear( order );
2184 static inline void atomic_thread_fence(memory_order order) CDS_NOEXCEPT
2186 platform::thread_fence( order );
2187 CDS_COMPILER_RW_BARRIER;
2189 static inline void atomic_signal_fence(memory_order order) CDS_NOEXCEPT
2191 platform::signal_fence( order );
2194 }} // namespace cds::cxx11_atomic
2197 #endif // #ifndef CDSLIB_COMPILER_CXX11_ATOMIC_H