X-Git-Url: http://plrg.eecs.uci.edu/git/?p=libcds.git;a=blobdiff_plain;f=src%2Fhp.cpp;h=616ea59bcda1ebe876e31c601d64d9331b803b54;hp=897e119a867f5b6139205b16967c0cacc527c958;hb=bc277979f85299cd681e5a413d63f0c7f1d67beb;hpb=aaf811669b8744b4a3562c725297ab7fe85a99cc diff --git a/src/hp.cpp b/src/hp.cpp index 897e119a..616ea59b 100644 --- a/src/hp.cpp +++ b/src/hp.cpp @@ -31,7 +31,7 @@ #include #include -#include +#include #include namespace cds { namespace gc { namespace hp { @@ -103,6 +103,8 @@ namespace cds { namespace gc { namespace hp { thread_record( guard* guards, size_t guard_count, retired_ptr* retired_arr, size_t retired_capacity ) : thread_data( guards, guard_count, retired_arr, retired_capacity ) + , m_pNextNode( nullptr ) + , m_idOwner( cds::OS::c_NullThreadId ) , m_bFree( false ) {} }; @@ -140,31 +142,30 @@ namespace cds { namespace gc { namespace hp { } CDS_EXPORT_API smr::smr( size_t nHazardPtrCount, size_t nMaxThreadCount, size_t nMaxRetiredPtrCount, scan_type nScanType ) - : thread_list_( nullptr ) - , hazard_ptr_count_( nHazardPtrCount == 0 ? defaults::c_nHazardPointerPerThread : nHazardPtrCount ) + : hazard_ptr_count_( nHazardPtrCount == 0 ? defaults::c_nHazardPointerPerThread : nHazardPtrCount ) , max_thread_count_( nMaxThreadCount == 0 ? defaults::c_nMaxThreadCount : nMaxThreadCount ) , max_retired_ptr_count_( calc_retired_size( nMaxRetiredPtrCount, hazard_ptr_count_, max_thread_count_ )) , scan_type_( nScanType ) , scan_func_( nScanType == classic ? &smr::classic_scan : &smr::inplace_scan ) - {} + { + thread_list_.store( nullptr, atomics::memory_order_release ); + } CDS_EXPORT_API smr::~smr() { CDS_DEBUG_ONLY( const cds::OS::ThreadId nullThreadId = cds::OS::c_NullThreadId; ) - CDS_DEBUG_ONLY( const cds::OS::ThreadId mainThreadId = cds::OS::get_current_thread_id();) + CDS_DEBUG_ONLY( const cds::OS::ThreadId mainThreadId = cds::OS::get_current_thread_id();) - CDS_HPSTAT( statistics( s_postmortem_stat )); + CDS_HPSTAT( statistics( s_postmortem_stat ) ); thread_record* pHead = thread_list_.load( atomics::memory_order_relaxed ); - thread_list_.store( nullptr, atomics::memory_order_relaxed ); + thread_list_.store( nullptr, atomics::memory_order_release ); thread_record* pNext = nullptr; for ( thread_record* hprec = pHead; hprec; hprec = pNext ) { assert( hprec->m_idOwner.load( atomics::memory_order_relaxed ) == nullThreadId - || hprec->m_idOwner.load( atomics::memory_order_relaxed ) == mainThreadId - || !cds::OS::is_thread_alive( hprec->m_idOwner.load( atomics::memory_order_relaxed ) ) - ); + || hprec->m_idOwner.load( atomics::memory_order_relaxed ) == mainThreadId ); retired_array& arr = hprec->retired_; for ( retired_ptr* cur{ arr.first() }, *last{ arr.last() }; cur != last; ++cur ) { @@ -206,10 +207,13 @@ namespace cds { namespace gc { namespace hp { +--------------------------+ */ - char* mem = reinterpret_cast( s_alloc_memory( nSize )); - return new( mem ) thread_record( - reinterpret_cast( mem + sizeof( thread_record )), get_hazard_ptr_count(), - reinterpret_cast( mem + sizeof( thread_record ) + guard_array_size ), get_max_retired_ptr_count() + uint8_t* mem = reinterpret_cast( s_alloc_memory( nSize )); + + return new( mem ) thread_record( + reinterpret_cast( mem + sizeof( thread_record )), + get_hazard_ptr_count(), + reinterpret_cast( mem + sizeof( thread_record ) + guard_array_size ), + get_max_retired_ptr_count() ); } @@ -225,16 +229,14 @@ namespace cds { namespace gc { namespace hp { CDS_EXPORT_API smr::thread_record* smr::alloc_thread_data() { - //CDS_HAZARDPTR_STATISTIC( ++m_Stat.m_AllocHPRec ) - thread_record * hprec; const cds::OS::ThreadId nullThreadId = cds::OS::c_NullThreadId; const cds::OS::ThreadId curThreadId = cds::OS::get_current_thread_id(); // First try to reuse a free (non-active) HP record - for ( hprec = thread_list_.load( atomics::memory_order_acquire ); hprec; hprec = hprec->m_pNextNode.load( atomics::memory_order_relaxed ) ) { + for ( hprec = thread_list_.load( atomics::memory_order_acquire ); hprec; hprec = hprec->m_pNextNode.load( atomics::memory_order_acquire )) { cds::OS::ThreadId thId = nullThreadId; - if ( !hprec->m_idOwner.compare_exchange_strong( thId, curThreadId, atomics::memory_order_relaxed, atomics::memory_order_relaxed ) ) + if ( !hprec->m_idOwner.compare_exchange_strong( thId, curThreadId, atomics::memory_order_relaxed, atomics::memory_order_relaxed )) continue; hprec->m_bFree.store( false, atomics::memory_order_release ); return hprec; @@ -247,7 +249,7 @@ namespace cds { namespace gc { namespace hp { thread_record* pOldHead = thread_list_.load( atomics::memory_order_relaxed ); do { - hprec->m_pNextNode.store( pOldHead, atomics::memory_order_relaxed ); + hprec->m_pNextNode.store( pOldHead, atomics::memory_order_release ); } while ( !thread_list_.compare_exchange_weak( pOldHead, hprec, atomics::memory_order_release, atomics::memory_order_acquire ) ); return hprec; @@ -256,7 +258,6 @@ namespace cds { namespace gc { namespace hp { CDS_EXPORT_API void smr::free_thread_data( smr::thread_record* pRec ) { assert( pRec != nullptr ); - //CDS_HAZARDPTR_STATISTIC( ++m_Stat.m_RetireHPRec ) pRec->hazards_.clear(); scan( pRec ); @@ -318,7 +319,7 @@ namespace cds { namespace gc { namespace hp { } } - CDS_HPSTAT( ++pRec->stat_.scan_count ); + CDS_HPSTAT( ++pThreadRec->scan_count_ ); // Sort retired pointer array std::sort( first_retired, last_retired, retired_ptr::less ); @@ -341,7 +342,7 @@ namespace cds { namespace gc { namespace hp { { retired_ptr dummy_retired; while ( pNode ) { - if ( !pNode->m_bFree.load( atomics::memory_order_relaxed )) { + if ( pNode->m_idOwner.load( atomics::memory_order_relaxed ) != cds::OS::c_NullThreadId ) { thread_hp_storage& hpstg = pNode->hazards_; for ( size_t i = 0; i < hazard_ptr_count_; ++i ) { pRec->sync(); @@ -373,7 +374,7 @@ namespace cds { namespace gc { namespace hp { else { // Retired pointer may be freed it->free(); - CDS_HPSTAT( ++pRec->stat_.free_count ); + CDS_HPSTAT( ++pRec->free_count_ ); } } const size_t nDeferred = insert_pos - first_retired; @@ -386,7 +387,7 @@ namespace cds { namespace gc { namespace hp { { thread_record* pRec = static_cast( pThreadRec ); - CDS_HPSTAT( ++pRec->stat_.scan_count ); + CDS_HPSTAT( ++pThreadRec->scan_count_ ); std::vector< void*, allocator> plist; plist.reserve( get_max_thread_count() * get_hazard_ptr_count()); @@ -397,11 +398,13 @@ namespace cds { namespace gc { namespace hp { thread_record* pNode = thread_list_.load( atomics::memory_order_acquire ); while ( pNode ) { - for ( size_t i = 0; i < get_hazard_ptr_count(); ++i ) { - pRec->sync(); - void * hptr = pNode->hazards_[i].get(); - if ( hptr ) - plist.push_back( hptr ); + if ( pNode->m_idOwner.load( std::memory_order_relaxed ) != cds::OS::c_NullThreadId ) { + for ( size_t i = 0; i < get_hazard_ptr_count(); ++i ) { + pRec->sync(); + void * hptr = pNode->hazards_[i].get(); + if ( hptr ) + plist.push_back( hptr ); + } } pNode = pNode->m_pNextNode.load( atomics::memory_order_relaxed ); } @@ -427,14 +430,11 @@ namespace cds { namespace gc { namespace hp { } else { it->free(); - CDS_HPSTAT( ++pRec->stat_.free_count ); + CDS_HPSTAT( ++pRec->free_count_ ); } } retired.reset( insert_pos - first_retired ); - - //CDS_HAZARDPTR_STATISTIC( m_Stat.m_DeferredNode += nDeferredCount ) - //CDS_HAZARDPTR_STATISTIC( m_Stat.m_DeletedNode += ( itRetiredEnd - arrRetired.begin() ) - nDeferredCount ) } } @@ -442,12 +442,15 @@ namespace cds { namespace gc { namespace hp { { assert( static_cast( pThis )->m_idOwner.load( atomics::memory_order_relaxed ) == cds::OS::get_current_thread_id() ); - CDS_HPSTAT( ++pThis->stat_.help_scan_count ); + CDS_HPSTAT( ++pThis->help_scan_count_ ); const cds::OS::ThreadId nullThreadId = cds::OS::c_NullThreadId; const cds::OS::ThreadId curThreadId = cds::OS::get_current_thread_id(); for ( thread_record* hprec = thread_list_.load( atomics::memory_order_acquire ); hprec; hprec = hprec->m_pNextNode.load( atomics::memory_order_relaxed )) { + if ( hprec == static_cast( pThis )) + continue; + // If m_bFree == true then hprec->retired_ is empty - we don't need to see it if ( hprec->m_bFree.load( atomics::memory_order_acquire )) continue; @@ -456,7 +459,7 @@ namespace cds { namespace gc { namespace hp { // Several threads may work concurrently so we use atomic technique only. { cds::OS::ThreadId curOwner = hprec->m_idOwner.load( atomics::memory_order_relaxed ); - if ( curOwner == nullThreadId || !cds::OS::is_thread_alive( curOwner ) ) { + if ( curOwner == nullThreadId ) { if ( !hprec->m_idOwner.compare_exchange_strong( curOwner, curThreadId, atomics::memory_order_acquire, atomics::memory_order_relaxed ) ) continue; } @@ -465,7 +468,7 @@ namespace cds { namespace gc { namespace hp { } // We own the thread record successfully. Now, we can see whether it has retired pointers. - // If it has ones then we move to pThis that is private for current thread. + // If it has ones then we move them to pThis that is private for current thread. retired_array& src = hprec->retired_; retired_array& dest = pThis->retired_; assert( !dest.full() ); @@ -478,35 +481,36 @@ namespace cds { namespace gc { namespace hp { scan( pThis ); } - src.reset( 0 ); - - hprec->m_bFree.store( true, atomics::memory_order_relaxed ); + src.interthread_clear(); + hprec->m_bFree.store( true, atomics::memory_order_release ); hprec->m_idOwner.store( nullThreadId, atomics::memory_order_release ); scan( pThis ); } } - void smr::statistics( stat& st ) + CDS_EXPORT_API void smr::statistics( stat& st ) { st.clear(); # ifdef CDS_ENABLE_HPSTAT for ( thread_record* hprec = thread_list_.load( atomics::memory_order_acquire ); hprec; hprec = hprec->m_pNextNode.load( atomics::memory_order_relaxed ) ) { + CDS_TSAN_ANNOTATE_IGNORE_READS_BEGIN; ++st.thread_rec_count; st.guard_allocated += hprec->hazards_.alloc_guard_count_; st.guard_freed += hprec->hazards_.free_guard_count_; st.retired_count += hprec->retired_.retire_call_count_; - st.free_count += hprec->stat_.free_count; - st.scan_count += hprec->stat_.scan_count; - st.help_scan_count += hprec->stat_.help_scan_count; + st.free_count += hprec->free_count_; + st.scan_count += hprec->scan_count_; + st.help_scan_count += hprec->help_scan_count_; + CDS_TSAN_ANNOTATE_IGNORE_READS_END; } # endif } }}} // namespace cds::gc::hp -/*static*/ cds::gc::HP::stat const& cds::gc::HP::postmortem_statistics() +CDS_EXPORT_API /*static*/ cds::gc::HP::stat const& cds::gc::HP::postmortem_statistics() { return cds::gc::hp::s_postmortem_stat; }