#include <algorithm>
#include <vector>
-#include <cds/gc/hp_smr.h>
+#include <cds/gc/hp.h>
#include <cds/os/thread.h>
namespace cds { namespace gc { namespace hp {
thread_record( guard* guards, size_t guard_count, retired_ptr* retired_arr, size_t retired_capacity )
: thread_data( guards, guard_count, retired_arr, retired_capacity )
+ , m_pNextNode( nullptr )
+ , m_idOwner( cds::OS::c_NullThreadId )
, m_bFree( false )
{}
};
}
CDS_EXPORT_API smr::smr( size_t nHazardPtrCount, size_t nMaxThreadCount, size_t nMaxRetiredPtrCount, scan_type nScanType )
- : thread_list_( nullptr )
- , hazard_ptr_count_( nHazardPtrCount == 0 ? defaults::c_nHazardPointerPerThread : nHazardPtrCount )
+ : hazard_ptr_count_( nHazardPtrCount == 0 ? defaults::c_nHazardPointerPerThread : nHazardPtrCount )
, max_thread_count_( nMaxThreadCount == 0 ? defaults::c_nMaxThreadCount : nMaxThreadCount )
, max_retired_ptr_count_( calc_retired_size( nMaxRetiredPtrCount, hazard_ptr_count_, max_thread_count_ ))
, scan_type_( nScanType )
, scan_func_( nScanType == classic ? &smr::classic_scan : &smr::inplace_scan )
- {}
+ {
+ thread_list_.store( nullptr, atomics::memory_order_release );
+ }
CDS_EXPORT_API smr::~smr()
{
CDS_DEBUG_ONLY( const cds::OS::ThreadId nullThreadId = cds::OS::c_NullThreadId; )
- CDS_DEBUG_ONLY( const cds::OS::ThreadId mainThreadId = cds::OS::get_current_thread_id();)
+ CDS_DEBUG_ONLY( const cds::OS::ThreadId mainThreadId = cds::OS::get_current_thread_id();)
- CDS_HPSTAT( statistics( s_postmortem_stat ));
+ CDS_HPSTAT( statistics( s_postmortem_stat ) );
thread_record* pHead = thread_list_.load( atomics::memory_order_relaxed );
- thread_list_.store( nullptr, atomics::memory_order_relaxed );
+ thread_list_.store( nullptr, atomics::memory_order_release );
thread_record* pNext = nullptr;
for ( thread_record* hprec = pHead; hprec; hprec = pNext )
{
assert( hprec->m_idOwner.load( atomics::memory_order_relaxed ) == nullThreadId
- || hprec->m_idOwner.load( atomics::memory_order_relaxed ) == mainThreadId
- || !cds::OS::is_thread_alive( hprec->m_idOwner.load( atomics::memory_order_relaxed ) )
- );
+ || hprec->m_idOwner.load( atomics::memory_order_relaxed ) == mainThreadId );
retired_array& arr = hprec->retired_;
for ( retired_ptr* cur{ arr.first() }, *last{ arr.last() }; cur != last; ++cur ) {
+--------------------------+
*/
- char* mem = reinterpret_cast<char*>( s_alloc_memory( nSize ));
- return new( mem ) thread_record(
- reinterpret_cast<guard*>( mem + sizeof( thread_record )), get_hazard_ptr_count(),
- reinterpret_cast<retired_ptr*>( mem + sizeof( thread_record ) + guard_array_size ), get_max_retired_ptr_count()
+ uint8_t* mem = reinterpret_cast<uint8_t*>( s_alloc_memory( nSize ));
+
+ return new( mem ) thread_record(
+ reinterpret_cast<guard*>( mem + sizeof( thread_record )),
+ get_hazard_ptr_count(),
+ reinterpret_cast<retired_ptr*>( mem + sizeof( thread_record ) + guard_array_size ),
+ get_max_retired_ptr_count()
);
}
CDS_EXPORT_API smr::thread_record* smr::alloc_thread_data()
{
- //CDS_HAZARDPTR_STATISTIC( ++m_Stat.m_AllocHPRec )
-
thread_record * hprec;
const cds::OS::ThreadId nullThreadId = cds::OS::c_NullThreadId;
const cds::OS::ThreadId curThreadId = cds::OS::get_current_thread_id();
// First try to reuse a free (non-active) HP record
- for ( hprec = thread_list_.load( atomics::memory_order_acquire ); hprec; hprec = hprec->m_pNextNode.load( atomics::memory_order_relaxed ) ) {
+ for ( hprec = thread_list_.load( atomics::memory_order_acquire ); hprec; hprec = hprec->m_pNextNode.load( atomics::memory_order_acquire )) {
cds::OS::ThreadId thId = nullThreadId;
- if ( !hprec->m_idOwner.compare_exchange_strong( thId, curThreadId, atomics::memory_order_relaxed, atomics::memory_order_relaxed ) )
+ if ( !hprec->m_idOwner.compare_exchange_strong( thId, curThreadId, atomics::memory_order_relaxed, atomics::memory_order_relaxed ))
continue;
hprec->m_bFree.store( false, atomics::memory_order_release );
return hprec;
thread_record* pOldHead = thread_list_.load( atomics::memory_order_relaxed );
do {
- hprec->m_pNextNode.store( pOldHead, atomics::memory_order_relaxed );
+ hprec->m_pNextNode.store( pOldHead, atomics::memory_order_release );
} while ( !thread_list_.compare_exchange_weak( pOldHead, hprec, atomics::memory_order_release, atomics::memory_order_acquire ) );
return hprec;
CDS_EXPORT_API void smr::free_thread_data( smr::thread_record* pRec )
{
assert( pRec != nullptr );
- //CDS_HAZARDPTR_STATISTIC( ++m_Stat.m_RetireHPRec )
pRec->hazards_.clear();
scan( pRec );
}
}
- CDS_HPSTAT( ++pRec->stat_.scan_count );
+ CDS_HPSTAT( ++pThreadRec->scan_count_ );
// Sort retired pointer array
std::sort( first_retired, last_retired, retired_ptr::less );
{
retired_ptr dummy_retired;
while ( pNode ) {
- if ( !pNode->m_bFree.load( atomics::memory_order_relaxed )) {
+ if ( pNode->m_idOwner.load( atomics::memory_order_relaxed ) != cds::OS::c_NullThreadId ) {
thread_hp_storage& hpstg = pNode->hazards_;
for ( size_t i = 0; i < hazard_ptr_count_; ++i ) {
pRec->sync();
else {
// Retired pointer may be freed
it->free();
- CDS_HPSTAT( ++pRec->stat_.free_count );
+ CDS_HPSTAT( ++pRec->free_count_ );
}
}
const size_t nDeferred = insert_pos - first_retired;
{
thread_record* pRec = static_cast<thread_record*>( pThreadRec );
- CDS_HPSTAT( ++pRec->stat_.scan_count );
+ CDS_HPSTAT( ++pThreadRec->scan_count_ );
std::vector< void*, allocator<void*>> plist;
plist.reserve( get_max_thread_count() * get_hazard_ptr_count());
thread_record* pNode = thread_list_.load( atomics::memory_order_acquire );
while ( pNode ) {
- for ( size_t i = 0; i < get_hazard_ptr_count(); ++i ) {
- pRec->sync();
- void * hptr = pNode->hazards_[i].get();
- if ( hptr )
- plist.push_back( hptr );
+ if ( pNode->m_idOwner.load( std::memory_order_relaxed ) != cds::OS::c_NullThreadId ) {
+ for ( size_t i = 0; i < get_hazard_ptr_count(); ++i ) {
+ pRec->sync();
+ void * hptr = pNode->hazards_[i].get();
+ if ( hptr )
+ plist.push_back( hptr );
+ }
}
pNode = pNode->m_pNextNode.load( atomics::memory_order_relaxed );
}
}
else {
it->free();
- CDS_HPSTAT( ++pRec->stat_.free_count );
+ CDS_HPSTAT( ++pRec->free_count_ );
}
}
retired.reset( insert_pos - first_retired );
-
- //CDS_HAZARDPTR_STATISTIC( m_Stat.m_DeferredNode += nDeferredCount )
- //CDS_HAZARDPTR_STATISTIC( m_Stat.m_DeletedNode += ( itRetiredEnd - arrRetired.begin() ) - nDeferredCount )
}
}
{
assert( static_cast<thread_record*>( pThis )->m_idOwner.load( atomics::memory_order_relaxed ) == cds::OS::get_current_thread_id() );
- CDS_HPSTAT( ++pThis->stat_.help_scan_count );
+ CDS_HPSTAT( ++pThis->help_scan_count_ );
const cds::OS::ThreadId nullThreadId = cds::OS::c_NullThreadId;
const cds::OS::ThreadId curThreadId = cds::OS::get_current_thread_id();
for ( thread_record* hprec = thread_list_.load( atomics::memory_order_acquire ); hprec; hprec = hprec->m_pNextNode.load( atomics::memory_order_relaxed ))
{
+ if ( hprec == static_cast<thread_record*>( pThis ))
+ continue;
+
// If m_bFree == true then hprec->retired_ is empty - we don't need to see it
if ( hprec->m_bFree.load( atomics::memory_order_acquire ))
continue;
// Several threads may work concurrently so we use atomic technique only.
{
cds::OS::ThreadId curOwner = hprec->m_idOwner.load( atomics::memory_order_relaxed );
- if ( curOwner == nullThreadId || !cds::OS::is_thread_alive( curOwner ) ) {
+ if ( curOwner == nullThreadId ) {
if ( !hprec->m_idOwner.compare_exchange_strong( curOwner, curThreadId, atomics::memory_order_acquire, atomics::memory_order_relaxed ) )
continue;
}
}
// We own the thread record successfully. Now, we can see whether it has retired pointers.
- // If it has ones then we move to pThis that is private for current thread.
+ // If it has ones then we move them to pThis that is private for current thread.
retired_array& src = hprec->retired_;
retired_array& dest = pThis->retired_;
assert( !dest.full() );
scan( pThis );
}
- src.reset( 0 );
-
- hprec->m_bFree.store( true, atomics::memory_order_relaxed );
+ src.interthread_clear();
+ hprec->m_bFree.store( true, atomics::memory_order_release );
hprec->m_idOwner.store( nullThreadId, atomics::memory_order_release );
scan( pThis );
}
}
- void smr::statistics( stat& st )
+ CDS_EXPORT_API void smr::statistics( stat& st )
{
st.clear();
# ifdef CDS_ENABLE_HPSTAT
for ( thread_record* hprec = thread_list_.load( atomics::memory_order_acquire ); hprec; hprec = hprec->m_pNextNode.load( atomics::memory_order_relaxed ) )
{
+ CDS_TSAN_ANNOTATE_IGNORE_READS_BEGIN;
++st.thread_rec_count;
st.guard_allocated += hprec->hazards_.alloc_guard_count_;
st.guard_freed += hprec->hazards_.free_guard_count_;
st.retired_count += hprec->retired_.retire_call_count_;
- st.free_count += hprec->stat_.free_count;
- st.scan_count += hprec->stat_.scan_count;
- st.help_scan_count += hprec->stat_.help_scan_count;
+ st.free_count += hprec->free_count_;
+ st.scan_count += hprec->scan_count_;
+ st.help_scan_count += hprec->help_scan_count_;
+ CDS_TSAN_ANNOTATE_IGNORE_READS_END;
}
# endif
}
}}} // namespace cds::gc::hp
-/*static*/ cds::gc::HP::stat const& cds::gc::HP::postmortem_statistics()
+CDS_EXPORT_API /*static*/ cds::gc::HP::stat const& cds::gc::HP::postmortem_statistics()
{
return cds::gc::hp::s_postmortem_stat;
}
projects / libcds.git / blobdiff