}
/// Initializes dummy node with \p nHash value
- hash_node( size_t nHash )
+ explicit hash_node( size_t nHash )
: m_nHash( nHash )
{
assert( is_dummy());
}
/// Initializes dummy node with \p nHash value
- node( size_t nHash )
+ explicit node( size_t nHash )
: hash_node( nHash )
{
assert( is_dummy());
}
/// Initializes dummy node with \p nHash value
- node( size_t nHash )
+ explicit node( size_t nHash )
: hash_node( nHash )
{
assert( is_dummy());
/// Auxiliary node type
struct aux_node_type: public node_type, public free_list::node
- {};
+ {
+# ifdef CDS_DEBUG
+ atomics::atomic<bool> m_busy;
+
+ aux_node_type()
+ {
+ m_busy.store( false, atomics::memory_order_release );
+ }
+# endif
+ };
typedef atomics::atomic<aux_node_type *> table_entry; ///< Table entry type
typedef cds::details::Allocator< table_entry, allocator > bucket_table_allocator; ///< Bucket table allocator
if ( m_nAuxNodeAllocated.load( memory_model::memory_order_relaxed ) < capacity()) {
// alloc next free node from m_auxNode
size_t const idx = m_nAuxNodeAllocated.fetch_add( 1, memory_model::memory_order_relaxed );
- if ( idx < capacity())
+ if ( idx < capacity() ) {
+ CDS_TSAN_ANNOTATE_NEW_MEMORY( &m_auxNode[idx], sizeof( aux_node_type ) );
return new( &m_auxNode[idx] ) aux_node_type();
+ }
}
// get from free-list
typedef typename options::free_list free_list;
/// Auxiliary node type
- class aux_node_type: public node_type, public free_list::node
- {};
+ struct aux_node_type: public node_type, public free_list::node
+ {
+# ifdef CDS_DEBUG
+ atomics::atomic<bool> m_busy;
+
+ aux_node_type()
+ {
+ m_busy.store( false, atomics::memory_order_release );
+ }
+# endif
+ };
protected:
//@cond
// aux_node_type nodes[];
aux_node_segment()
- : aux_node_count(0)
- , next_segment( nullptr )
- {}
+ : next_segment( nullptr )
+ {
+ aux_node_count.store( 0, atomics::memory_order_release );
+ }
aux_node_type* segment()
{
assert( aux_segment != nullptr );
// try to allocate from current aux segment
- if ( aux_segment->aux_node_count.load( memory_model::memory_order_relaxed ) < m_metrics.nSegmentSize ) {
+ if ( aux_segment->aux_node_count.load( memory_model::memory_order_acquire ) < m_metrics.nSegmentSize ) {
size_t idx = aux_segment->aux_node_count.fetch_add( 1, memory_model::memory_order_relaxed );
- if ( idx < m_metrics.nSegmentSize )
+ if ( idx < m_metrics.nSegmentSize ) {
+ CDS_TSAN_ANNOTATE_NEW_MEMORY( aux_segment->segment() + idx, sizeof( aux_node_type ) );
return new( aux_segment->segment() + idx ) aux_node_type();
+ }
}
// try allocate from free-list
aux_node_segment* new_aux_segment = allocate_aux_segment();
new_aux_segment->next_segment = aux_segment;
new_aux_segment->aux_node_count.fetch_add( 1, memory_model::memory_order_relaxed );
- CDS_COMPILER_RW_BARRIER;
- if ( m_auxNodeList.compare_exchange_strong( aux_segment, new_aux_segment, memory_model::memory_order_release, atomics::memory_order_acquire ))
- return new( new_aux_segment->segment()) aux_node_type();
+ if ( m_auxNodeList.compare_exchange_strong( aux_segment, new_aux_segment, memory_model::memory_order_release, atomics::memory_order_acquire ) ) {
+ CDS_TSAN_ANNOTATE_NEW_MEMORY( new_aux_segment->segment(), sizeof( aux_node_type ) );
+ return new( new_aux_segment->segment() ) aux_node_type();
+ }
free_aux_segment( new_aux_segment );
}
aux_node_segment* allocate_aux_segment()
{
char* p = raw_allocator().allocate( sizeof( aux_node_segment ) + sizeof( aux_node_type ) * m_metrics.nSegmentSize );
+ CDS_TSAN_ANNOTATE_NEW_MEMORY( p, sizeof( aux_node_segment ) );
return new(p) aux_node_segment();
}
atomics::atomic<node *> m_freeListNext;
node()
- : m_freeListNext( nullptr )
- {}
+ {
+ m_freeListNext.store( nullptr, atomics::memory_order_release );
+ }
//@endcond
};
do {
newHead.tag = currentHead.tag + 1;
pNode->m_freeListNext.store( currentHead.ptr, atomics::memory_order_relaxed );
- } while ( cds_unlikely( !m_Head.compare_exchange_weak( currentHead, newHead, atomics::memory_order_release, atomics::memory_order_relaxed )));
+ CDS_TSAN_ANNOTATE_HAPPENS_BEFORE( &pNode->m_freeListNext );
+ } while ( cds_unlikely( !m_Head.compare_exchange_weak( currentHead, newHead, atomics::memory_order_release, atomics::memory_order_acquire )));
}
/// Gets a node from the free list. If the list is empty, returns \p nullptr
tagged_ptr currentHead = m_Head.load( atomics::memory_order_acquire );
tagged_ptr newHead;
while ( currentHead.ptr != nullptr ) {
+ CDS_TSAN_ANNOTATE_HAPPENS_AFTER( ¤tHead.ptr->m_freeListNext );
newHead.ptr = currentHead.ptr->m_freeListNext.load( atomics::memory_order_relaxed );
newHead.tag = currentHead.tag + 1;
if ( cds_likely( m_Head.compare_exchange_weak( currentHead, newHead, atomics::memory_order_release, atomics::memory_order_acquire )))
{
m_Stat.onHeadNodeAllocated();
aux_node_type* p = m_Buckets.alloc_aux_node();
- if ( p )
+ if ( p ) {
+ CDS_TSAN_ANNOTATE_IGNORE_WRITES_BEGIN;
+ // p->m_nHash is read-only data member
p->m_nHash = nHash;
+ CDS_TSAN_ANNOTATE_IGNORE_WRITES_END;
+# ifdef CDS_DEBUG
+ cds_assert( !p->m_busy.load( atomics::memory_order_acquire ) );
+ p->m_busy.store( true, atomics::memory_order_release );
+# endif
+ }
return p;
}
void free_aux_node( aux_node_type * p )
{
+# ifdef CDS_DEBUG
+ cds_assert( p->m_busy.load( atomics::memory_order_acquire ) );
+ p->m_busy.store( false, atomics::memory_order_release );
+# endif
+
m_Buckets.free_aux_node( p );
m_Stat.onHeadNodeFreed();
}