// allocate new block
rb = new( s_alloc_memory( sizeof( retired_block ) + sizeof( retired_ptr ) * retired_block::c_capacity )) retired_block;
new ( rb->first()) retired_ptr[retired_block::c_capacity];
// allocate new block
rb = new( s_alloc_memory( sizeof( retired_block ) + sizeof( retired_ptr ) * retired_block::c_capacity )) retired_block;
new ( rb->first()) retired_ptr[retired_block::c_capacity];
thread_record( guard* guards, size_t guard_count )
: thread_data( guards, guard_count )
thread_record( guard* guards, size_t guard_count )
: thread_data( guards, guard_count )
CDS_EXPORT_API smr::~smr()
{
CDS_DEBUG_ONLY( const cds::OS::ThreadId nullThreadId = cds::OS::c_NullThreadId; )
CDS_DEBUG_ONLY( const cds::OS::ThreadId mainThreadId = cds::OS::get_current_thread_id(); )
CDS_EXPORT_API smr::~smr()
{
CDS_DEBUG_ONLY( const cds::OS::ThreadId nullThreadId = cds::OS::c_NullThreadId; )
CDS_DEBUG_ONLY( const cds::OS::ThreadId mainThreadId = cds::OS::get_current_thread_id(); )
thread_record* pNext = nullptr;
for ( thread_record* hprec = pHead; hprec; hprec = pNext )
{
assert( hprec->m_idOwner.load( atomics::memory_order_relaxed ) == nullThreadId
thread_record* pNext = nullptr;
for ( thread_record* hprec = pHead; hprec; hprec = pNext )
{
assert( hprec->m_idOwner.load( atomics::memory_order_relaxed ) == nullThreadId
char* mem = reinterpret_cast<char*>( s_alloc_memory( sizeof( thread_record ) + guard_array_size ));
return new( mem ) thread_record(
char* mem = reinterpret_cast<char*>( s_alloc_memory( sizeof( thread_record ) + guard_array_size ));
return new( mem ) thread_record(
const cds::OS::ThreadId curThreadId = cds::OS::get_current_thread_id();
// First try to reuse a free (non-active) DHP record
const cds::OS::ThreadId curThreadId = cds::OS::get_current_thread_id();
// First try to reuse a free (non-active) DHP record
- if ( !hprec->m_idOwner.compare_exchange_strong( thId, curThreadId, atomics::memory_order_relaxed, atomics::memory_order_relaxed ) )
+ if ( !hprec->m_idOwner.compare_exchange_strong( thId, curThreadId, atomics::memory_order_relaxed, atomics::memory_order_relaxed ))
if ( !hprec ) {
// No HP records available for reuse
// Allocate and push a new HP record
hprec = create_thread_data();
hprec->m_idOwner.store( curThreadId, atomics::memory_order_relaxed );
if ( !hprec ) {
// No HP records available for reuse
// Allocate and push a new HP record
hprec = create_thread_data();
hprec->m_idOwner.store( curThreadId, atomics::memory_order_relaxed );
- hprec->m_pNextNode.store( pOldHead, atomics::memory_order_relaxed );
- } while ( !thread_list_.compare_exchange_weak( pOldHead, hprec, atomics::memory_order_release, atomics::memory_order_acquire ) );
+ hprec->m_pNextNode.store( pOldHead, atomics::memory_order_release );
+ } while ( !thread_list_.compare_exchange_weak( pOldHead, hprec, atomics::memory_order_release, atomics::memory_order_acquire ));
if ( pNode->m_idOwner.load( std::memory_order_relaxed ) != cds::OS::c_NullThreadId ) {
copy_hazards( plist, pNode->hazards_.array_, pNode->hazards_.initial_capacity_ );
if ( pNode->m_idOwner.load( std::memory_order_relaxed ) != cds::OS::c_NullThreadId ) {
copy_hazards( plist, pNode->hazards_.array_, pNode->hazards_.initial_capacity_ );
last_plist_size_.compare_exchange_weak( plist_size, plist.size(), std::memory_order_relaxed, std::memory_order_relaxed );
// Sort plist to simplify search in
last_plist_size_.compare_exchange_weak( plist_size, plist.size(), std::memory_order_relaxed, std::memory_order_relaxed );
// Sort plist to simplify search in
CDS_HPSTAT( pRec->free_call_count_ += free_count );
// If the count of freed elements is too small, increase retired array
CDS_HPSTAT( pRec->free_call_count_ += free_count );
// If the count of freed elements is too small, increase retired array
CDS_HPSTAT( ++pThis->help_scan_call_count_ );
const cds::OS::ThreadId nullThreadId = cds::OS::c_NullThreadId;
const cds::OS::ThreadId curThreadId = cds::OS::get_current_thread_id();
CDS_HPSTAT( ++pThis->help_scan_call_count_ );
const cds::OS::ThreadId nullThreadId = cds::OS::c_NullThreadId;
const cds::OS::ThreadId curThreadId = cds::OS::get_current_thread_id();
{
if ( hprec == static_cast<thread_record*>( pThis ))
continue;
// If m_bFree == true then hprec->retired_ is empty - we don't need to see it
{
if ( hprec == static_cast<thread_record*>( pThis ))
continue;
// If m_bFree == true then hprec->retired_ is empty - we don't need to see it
- if ( hprec->m_bFree.load( atomics::memory_order_acquire ) ) {
- assert( hprec->retired_.empty() );
+ if ( hprec->m_bFree.load( atomics::memory_order_acquire )) {
+ CDS_TSAN_ANNOTATE_IGNORE_READS_BEGIN;
+ assert( hprec->retired_.empty());
+ CDS_TSAN_ANNOTATE_IGNORE_READS_END;
{
cds::OS::ThreadId curOwner = hprec->m_idOwner.load( atomics::memory_order_relaxed );
if ( curOwner == nullThreadId ) {
{
cds::OS::ThreadId curOwner = hprec->m_idOwner.load( atomics::memory_order_relaxed );
if ( curOwner == nullThreadId ) {
- if ( !hprec->m_idOwner.compare_exchange_strong( curOwner, curThreadId, atomics::memory_order_acquire, atomics::memory_order_relaxed ) )
+ if ( !hprec->m_idOwner.compare_exchange_strong( curOwner, curThreadId, atomics::memory_order_acquire, atomics::memory_order_relaxed ))
for ( retired_block* block = src.list_head_; block; block = block->next_ ) {
retired_ptr* last = block == src.current_block_ ? src.current_cell_ : block->last();
for ( retired_ptr* p = block->first(); p != last; ++p ) {
for ( retired_block* block = src.list_head_; block; block = block->next_ ) {
retired_ptr* last = block == src.current_block_ ? src.current_cell_ : block->last();
for ( retired_ptr* p = block->first(); p != last; ++p ) {
++st.thread_rec_count;
st.guard_allocated += hprec->hazards_.alloc_guard_count_;
st.guard_freed += hprec->hazards_.free_guard_count_;
++st.thread_rec_count;
st.guard_allocated += hprec->hazards_.alloc_guard_count_;
st.guard_freed += hprec->hazards_.free_guard_count_;
st.free_count += hprec->free_call_count_;
st.scan_count += hprec->scan_call_count_;
st.help_scan_count += hprec->help_scan_call_count_;
st.free_count += hprec->free_call_count_;
st.scan_count += hprec->scan_call_count_;
st.help_scan_count += hprec->help_scan_call_count_;
st.hp_block_count = hp_allocator_.block_allocated_.load( atomics::memory_order_relaxed );
st.retired_block_count = retired_allocator_.block_allocated_.load( atomics::memory_order_relaxed );
st.hp_block_count = hp_allocator_.block_allocated_.load( atomics::memory_order_relaxed );
st.retired_block_count = retired_allocator_.block_allocated_.load( atomics::memory_order_relaxed );