3 #ifndef __CDS_COMPILER_VC_AMD64_CXX11_ATOMIC_H
4 #define __CDS_COMPILER_VC_AMD64_CXX11_ATOMIC_H
7 #include <emmintrin.h> // for 128bit atomic load/store
8 #include <cds/details/is_aligned.h>
10 #pragma intrinsic( _InterlockedIncrement )
11 #pragma intrinsic( _InterlockedDecrement )
12 #pragma intrinsic( _InterlockedCompareExchange )
13 #pragma intrinsic( _InterlockedCompareExchangePointer )
14 #pragma intrinsic( _InterlockedCompareExchange16 )
15 #pragma intrinsic( _InterlockedCompareExchange64 )
16 #pragma intrinsic( _InterlockedExchange )
17 #pragma intrinsic( _InterlockedExchange64 )
18 #pragma intrinsic( _InterlockedExchangePointer )
19 #pragma intrinsic( _InterlockedExchangeAdd )
20 #pragma intrinsic( _InterlockedExchangeAdd64 )
21 //#pragma intrinsic( _InterlockedAnd )
22 //#pragma intrinsic( _InterlockedOr )
23 //#pragma intrinsic( _InterlockedXor )
24 //#pragma intrinsic( _InterlockedAnd64 )
25 //#pragma intrinsic( _InterlockedOr64 )
26 //#pragma intrinsic( _InterlockedXor64 )
27 #pragma intrinsic( _interlockedbittestandset )
29 # pragma intrinsic( _InterlockedCompareExchange8 )
30 # pragma intrinsic( _InterlockedExchange8 )
31 # pragma intrinsic( _InterlockedExchange16 )
35 namespace cds { namespace cxx11_atomics {
36 namespace platform { CDS_CXX11_INLINE_NAMESPACE namespace vc { CDS_CXX11_INLINE_NAMESPACE namespace amd64 {
38 static inline void fence_before( memory_order order ) CDS_NOEXCEPT
41 case memory_order_relaxed:
42 case memory_order_acquire:
43 case memory_order_consume:
45 case memory_order_release:
46 case memory_order_acq_rel:
47 CDS_COMPILER_RW_BARRIER;
49 case memory_order_seq_cst:
50 CDS_COMPILER_RW_BARRIER;
55 static inline void fence_after( memory_order order ) CDS_NOEXCEPT
58 case memory_order_acquire:
59 case memory_order_acq_rel:
60 CDS_COMPILER_RW_BARRIER;
62 case memory_order_relaxed:
63 case memory_order_consume:
64 case memory_order_release:
66 case memory_order_seq_cst:
67 CDS_COMPILER_RW_BARRIER;
72 static inline void full_fence()
74 // MS VC does not support inline assembler in C code.
75 // So, we use InterlockedExchange for full fence instead of mfence inst
77 _InterlockedExchange( &t, 0 );
80 static inline void fence_after_load(memory_order order) CDS_NOEXCEPT
83 case memory_order_relaxed:
84 case memory_order_release:
86 case memory_order_acquire:
87 case memory_order_acq_rel:
88 CDS_COMPILER_RW_BARRIER;
90 case memory_order_consume:
92 case memory_order_seq_cst:
99 //-----------------------------------------------------------------------------
101 //-----------------------------------------------------------------------------
102 static inline void thread_fence(memory_order order) CDS_NOEXCEPT
106 case memory_order_relaxed:
107 case memory_order_consume:
109 case memory_order_release:
110 case memory_order_acquire:
111 case memory_order_acq_rel:
112 CDS_COMPILER_RW_BARRIER;
114 case memory_order_seq_cst:
121 static inline void signal_fence(memory_order order) CDS_NOEXCEPT
123 // C++11: 29.8.8: only compiler optimization, no hardware instructions
126 case memory_order_relaxed:
128 case memory_order_consume:
129 case memory_order_release:
130 case memory_order_acquire:
131 case memory_order_acq_rel:
132 case memory_order_seq_cst:
133 CDS_COMPILER_RW_BARRIER;
139 //-----------------------------------------------------------------------------
140 // atomic flag primitives
141 //-----------------------------------------------------------------------------
143 typedef unsigned char atomic_flag_type;
144 static inline bool atomic_flag_tas( atomic_flag_type volatile * pFlag, memory_order /*order*/ ) CDS_NOEXCEPT
146 return _interlockedbittestandset( (long volatile *) pFlag, 0 ) != 0;
149 static inline void atomic_flag_clear( atomic_flag_type volatile * pFlag, memory_order order ) CDS_NOEXCEPT
151 assert( order != memory_order_acquire
152 && order != memory_order_acq_rel
155 fence_before( order );
157 fence_after( order );
160 //-----------------------------------------------------------------------------
162 //-----------------------------------------------------------------------------
165 # pragma warning(push)
166 // Disable warning C4800: 'char' : forcing value to bool 'true' or 'false' (performance warning)
167 # pragma warning( disable: 4800 )
169 template <typename T>
170 static inline bool cas8_strong( T volatile * pDest, T& expected, T desired, memory_order /*mo_success*/, memory_order /*mo_fail*/ ) CDS_NOEXCEPT
172 static_assert( sizeof(T) == 1, "Illegal size of operand" );
174 # if _MSC_VER >= 1600
177 expected = (T) _InterlockedCompareExchange8( (char volatile*) pDest, (char) desired, (char) expected );
178 return expected == prev;
181 unsigned int * pnDest = (unsigned int *)( ((unsigned __int64) pDest) & ~(unsigned __int64(3)) );
182 unsigned int nOffset = ((unsigned __int64) pDest) & 3;
183 unsigned int nExpected;
184 unsigned int nDesired;
189 memcpy( reinterpret_cast<T *>(&nExpected) + nOffset, &expected, sizeof(T));
190 memcpy( reinterpret_cast<T *>(&nDesired) + nOffset, &desired, sizeof(T));
192 unsigned int nPrev = (unsigned int) _InterlockedCompareExchange( (long *) pnDest, (long) nDesired, (long) nExpected );
193 if ( nPrev == nExpected )
196 memcpy( &nByte, reinterpret_cast<T *>(&nPrev) + nOffset, sizeof(T));
197 if ( nByte != expected ) {
205 # pragma warning(pop)
208 template <typename T>
209 static inline bool cas8_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
211 return cas8_strong( pDest, expected, desired, mo_success, mo_fail );
215 # pragma warning(push)
216 // Disable warning C4800: 'char' : forcing value to bool 'true' or 'false' (performance warning)
217 # pragma warning( disable: 4800 )
219 template <typename T>
220 static inline T exchange8( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
222 static_assert( sizeof(T) == 1, "Illegal size of operand" );
224 # if _MSC_VER >= 1600
226 return (T) _InterlockedExchange8( (char volatile *) pDest, (char) v );
229 do {} while ( !cas8_strong( pDest, expected, v, order, memory_order_relaxed ));
234 # pragma warning(pop)
237 template <typename T>
238 static inline void store8( T volatile * pDest, T src, memory_order order ) CDS_NOEXCEPT
240 static_assert( sizeof(T) == 1, "Illegal size of operand" );
241 assert( order == memory_order_relaxed
242 || order == memory_order_release
243 || order == memory_order_seq_cst
245 assert( pDest != NULL );
247 if ( order != memory_order_seq_cst ) {
248 fence_before( order );
252 exchange8( pDest, src, order );
256 template <typename T>
257 static inline T load8( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
259 static_assert( sizeof(T) == 1, "Illegal size of operand" );
260 assert( order == memory_order_relaxed
261 || order == memory_order_consume
262 || order == memory_order_acquire
263 || order == memory_order_seq_cst
265 assert( pSrc != NULL );
268 fence_after_load( order );
272 //-----------------------------------------------------------------------------
274 //-----------------------------------------------------------------------------
276 template <typename T>
277 static inline bool cas16_strong( T volatile * pDest, T& expected, T desired, memory_order /*mo_success*/, memory_order /*mo_fail*/ ) CDS_NOEXCEPT
279 static_assert( sizeof(T) == 2, "Illegal size of operand" );
280 assert( cds::details::is_aligned( pDest, 2 ));
282 // _InterlockedCompareExchange behave as read-write memory barriers
284 expected = (T) _InterlockedCompareExchange16( (short *) pDest, (short) desired, (short) expected );
285 return expected == prev;
288 template <typename T>
289 static inline bool cas16_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
291 return cas16_strong( pDest, expected, desired, mo_success, mo_fail );
294 template <typename T>
295 static inline T exchange16( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
297 static_assert( sizeof(T) == 2, "Illegal size of operand" );
298 assert( cds::details::is_aligned( pDest, 2 ));
300 # if _MSC_VER >= 1600
302 return (T) _InterlockedExchange16( (short volatile *) pDest, (short) v );
305 do {} while ( !cas16_strong( pDest, expected, v, order, memory_order_relaxed ));
310 template <typename T>
311 static inline void store16( T volatile * pDest, T src, memory_order order ) CDS_NOEXCEPT
313 static_assert( sizeof(T) == 2, "Illegal size of operand" );
314 assert( order == memory_order_relaxed
315 || order == memory_order_release
316 || order == memory_order_seq_cst
318 assert( pDest != NULL );
319 assert( cds::details::is_aligned( pDest, 2 ));
321 if ( order != memory_order_seq_cst ) {
322 fence_before( order );
326 exchange16( pDest, src, order );
330 template <typename T>
331 static inline T load16( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
333 static_assert( sizeof(T) == 2, "Illegal size of operand" );
334 assert( order == memory_order_relaxed
335 || order == memory_order_consume
336 || order == memory_order_acquire
337 || order == memory_order_seq_cst
339 assert( pSrc != NULL );
340 assert( cds::details::is_aligned( pSrc, 2 ));
343 fence_after_load( order );
347 //-----------------------------------------------------------------------------
349 //-----------------------------------------------------------------------------
351 template <typename T>
352 static inline T exchange32( T volatile * pDest, T v, memory_order /*order*/ ) CDS_NOEXCEPT
354 static_assert( sizeof(T) == 4, "Illegal size of operand" );
355 assert( cds::details::is_aligned( pDest, 4 ));
357 return (T) _InterlockedExchange( (long *) pDest, (long) v );
360 template <typename T>
361 static inline void store32( T volatile * pDest, T src, memory_order order ) CDS_NOEXCEPT
363 static_assert( sizeof(T) == 4, "Illegal size of operand" );
364 assert( order == memory_order_relaxed
365 || order == memory_order_release
366 || order == memory_order_seq_cst
368 assert( pDest != NULL );
369 assert( cds::details::is_aligned( pDest, 4 ));
371 if ( order != memory_order_seq_cst ) {
372 fence_before( order );
376 exchange32( pDest, src, order );
380 template <typename T>
381 static inline T load32( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
383 static_assert( sizeof(T) == 4, "Illegal size of operand" );
384 assert( order == memory_order_relaxed
385 || order == memory_order_consume
386 || order == memory_order_acquire
387 || order == memory_order_seq_cst
389 assert( pSrc != NULL );
390 assert( cds::details::is_aligned( pSrc, 4 ));
393 fence_after_load( order );
397 template <typename T>
398 static inline bool cas32_strong( T volatile * pDest, T& expected, T desired, memory_order /*mo_success*/, memory_order /*mo_fail*/ ) CDS_NOEXCEPT
400 static_assert( sizeof(T) == 4, "Illegal size of operand" );
401 assert( cds::details::is_aligned( pDest, 4 ));
403 // _InterlockedCompareExchange behave as read-write memory barriers
405 expected = (T) _InterlockedCompareExchange( (long *) pDest, (long) desired, (long) expected );
406 return expected == prev;
409 template <typename T>
410 static inline bool cas32_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
412 return cas32_strong( pDest, expected, desired, mo_success, mo_fail );
415 // fetch_xxx may be emulated via cas32
416 // If the platform has special fetch_xxx instruction
417 // then it should define CDS_ATOMIC_fetch32_xxx_defined macro
419 # define CDS_ATOMIC_fetch32_add_defined
420 template <typename T>
421 static inline T fetch32_add( T volatile * pDest, T v, memory_order /*order*/) CDS_NOEXCEPT
423 static_assert( sizeof(T) == 4, "Illegal size of operand" );
424 assert( cds::details::is_aligned( pDest, 4 ));
426 // _InterlockedExchangeAdd behave as read-write memory barriers
427 return (T) _InterlockedExchangeAdd( (long *) pDest, (long) v );
430 //-----------------------------------------------------------------------------
432 //-----------------------------------------------------------------------------
434 template <typename T>
435 static inline bool cas64_strong( T volatile * pDest, T& expected, T desired, memory_order /*mo_success*/, memory_order /*mo_fail*/ ) CDS_NOEXCEPT
437 static_assert( sizeof(T) == 8, "Illegal size of operand" );
438 assert( cds::details::is_aligned( pDest, 8 ));
440 // _InterlockedCompareExchange behave as read-write memory barriers
442 expected = (T) _InterlockedCompareExchange64( (__int64 *) pDest, (__int64) desired, (__int64) expected );
443 return expected == prev;
446 template <typename T>
447 static inline bool cas64_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
449 return cas64_strong( pDest, expected, desired, mo_success, mo_fail );
452 template <typename T>
453 static inline T load64( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
455 static_assert( sizeof(T) == 8, "Illegal size of operand" );
456 assert( order == memory_order_relaxed
457 || order == memory_order_consume
458 || order == memory_order_acquire
459 || order == memory_order_seq_cst
461 assert( pSrc != NULL );
462 assert( cds::details::is_aligned( pSrc, 8 ));
465 fence_after_load( order );
470 template <typename T>
471 static inline T exchange64( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
473 static_assert( sizeof(T) == 8, "Illegal size of operand" );
475 T cur = load64( pDest, memory_order_relaxed );
477 } while (!cas64_weak( pDest, cur, v, order, memory_order_relaxed ));
481 template <typename T>
482 static inline void store64( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
484 static_assert( sizeof(T) == 8, "Illegal size of operand" );
485 assert( order == memory_order_relaxed
486 || order == memory_order_release
487 || order == memory_order_seq_cst
489 assert( pDest != NULL );
490 assert( cds::details::is_aligned( pDest, 8 ));
492 if ( order != memory_order_seq_cst ) {
493 fence_before( order );
497 exchange64( pDest, val, order );
501 # define CDS_ATOMIC_fetch64_add_defined
502 template <typename T>
503 static inline T fetch64_add( T volatile * pDest, T v, memory_order /*order*/) CDS_NOEXCEPT
505 static_assert( sizeof(T) == 8, "Illegal size of operand" );
506 assert( cds::details::is_aligned( pDest, 8 ));
508 // _InterlockedExchangeAdd64 behave as read-write memory barriers
509 return (T) _InterlockedExchangeAdd64( (__int64 *) pDest, (__int64) v );
512 //-----------------------------------------------------------------------------
513 // pointer primitives
514 //-----------------------------------------------------------------------------
516 template <typename T>
517 static inline T * exchange_ptr( T * volatile * pDest, T * v, memory_order /*order*/ ) CDS_NOEXCEPT
519 static_assert( sizeof(T *) == sizeof(void *), "Illegal size of operand" );
520 return (T *) _InterlockedExchangePointer( (void * volatile *) pDest, reinterpret_cast<void *>(v) );
523 template <typename T>
524 static inline void store_ptr( T * volatile * pDest, T * src, memory_order order ) CDS_NOEXCEPT
526 static_assert( sizeof(T *) == sizeof(void *), "Illegal size of operand" );
527 assert( order == memory_order_relaxed
528 || order == memory_order_release
529 || order == memory_order_seq_cst
531 assert( pDest != NULL );
533 if ( order != memory_order_seq_cst ) {
534 fence_before( order );
538 exchange_ptr( pDest, src, order );
542 template <typename T>
543 static inline T * load_ptr( T * volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
545 static_assert( sizeof(T *) == sizeof(void *), "Illegal size of operand" );
546 assert( order == memory_order_relaxed
547 || order == memory_order_consume
548 || order == memory_order_acquire
549 || order == memory_order_seq_cst
551 assert( pSrc != NULL );
554 fence_after_load( order );
558 template <typename T>
559 static inline bool cas_ptr_strong( T * volatile * pDest, T *& expected, T * desired, memory_order /*mo_success*/, memory_order /*mo_fail*/ ) CDS_NOEXCEPT
561 static_assert( sizeof(T *) == sizeof(void *), "Illegal size of operand" );
563 // _InterlockedCompareExchangePointer behave as read-write memory barriers
565 expected = (T *) _InterlockedCompareExchangePointer( (void * volatile *) pDest, (void *) desired, (void *) expected );
566 return expected == prev;
569 template <typename T>
570 static inline bool cas_ptr_weak( T * volatile * pDest, T *& expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
572 return cas_ptr_strong( pDest, expected, desired, mo_success, mo_fail );
575 }} // namespace vc::amd64
577 #ifndef CDS_CXX11_INLINE_NAMESPACE_SUPPORT
578 using namespace vc::amd64;
580 } // namespace platform
581 }} // namespace cds::cxx11_atomics
584 #endif // #ifndef __CDS_COMPILER_VC_AMD64_CXX11_ATOMIC_H