#include <threads.h>
#include <stdlib.h>
#include "librace.h"
+#include "model-assert.h"
#include "my_queue.h"
#define release memory_order_release
#define acquire memory_order_acquire
-static unsigned int *node_nums;
+#define MAX_FREELIST 4 /* Each thread can own up to MAX_FREELIST free nodes */
+#define INITIAL_FREE 2 /* Each thread starts with INITIAL_FREE free nodes */
+#define POISON_IDX 0x666
+
+static unsigned int (*free_lists)[MAX_FREELIST];
+
+/* Search this thread's free list for a "new" node */
static unsigned int new_node()
{
- return node_nums[get_thread_num()];
+ int i;
+ int t = get_thread_num();
+ for (i = 0; i < MAX_FREELIST; i++) {
+ unsigned int node = load_32(&free_lists[t][i]);
+ if (node) {
+ store_32(&free_lists[t][i], 0);
+ return node;
+ }
+ }
+ /* free_list is empty? */
+ MODEL_ASSERT(0);
+ return 0;
}
+/* Place this node index back on this thread's free list */
static void reclaim(unsigned int node)
{
- node_nums[get_thread_num()] = node;
-}
+ int i;
+ int t = get_thread_num();
-void init_queue(queue_t *q, int num_threads)
-{
- unsigned int i;
- pointer head;
- pointer tail;
- pointer next;
+ /* Don't reclaim NULL node */
+ MODEL_ASSERT(node);
- node_nums = malloc(num_threads * sizeof(*node_nums));
- for (i = 0; i < num_threads; i++)
- node_nums[i] = 2 + i;
+ for (i = 0; i < MAX_FREELIST; i++) {
+ /* Should never race with our own thread here */
+ unsigned int idx = load_32(&free_lists[t][i]);
- /* Note: needed to add this init manually */
- atomic_init(&q->nodes[0].next, 0);
+ /* Found empty spot in free list */
+ if (idx == 0) {
+ store_32(&free_lists[t][i], node);
+ return;
+ }
+ }
+ /* free list is full? */
+ MODEL_ASSERT(0);
+}
- /* initialize queue */
- head = MAKE_POINTER(1, 0);
- tail = MAKE_POINTER(1, 0);
- next = MAKE_POINTER(0, 0); // (NULL, 0)
-
- atomic_init(&q->head, head);
- atomic_init(&q->tail, tail);
- atomic_init(&q->nodes[1].next, next);
-
- /* initialize avail list */
- for (i = 2; i < MAX_NODES; i++) {
- next = MAKE_POINTER(i + 1, 0);
- atomic_init(&q->nodes[i].next, next);
+void init_queue(queue_t *q, int num_threads)
+{
+ int i, j;
+
+ /* Initialize each thread's free list with INITIAL_FREE pointers */
+ /* The actual nodes are initialized with poison indexes */
+ free_lists = malloc(num_threads * sizeof(*free_lists));
+ for (i = 0; i < num_threads; i++) {
+ for (j = 0; j < INITIAL_FREE; j++) {
+ free_lists[i][j] = 2 + i * MAX_FREELIST + j;
+ atomic_init(&q->nodes[free_lists[i][j]].next, MAKE_POINTER(POISON_IDX, 0));
+ }
}
- next = MAKE_POINTER(0, 0); // (NULL, 0)
- atomic_init(&q->nodes[MAX_NODES].next, next);
+ /* initialize queue */
+ atomic_init(&q->head, MAKE_POINTER(1, 0));
+ atomic_init(&q->tail, MAKE_POINTER(1, 0));
+ atomic_init(&q->nodes[1].next, MAKE_POINTER(0, 0));
}
void enqueue(queue_t *q, unsigned int val)
pointer next;
pointer tmp;
- /* Get a fresh node, set its value field, then set it's next reference to null */
node = new_node();
store_32(&q->nodes[node].value, val);
tmp = atomic_load_explicit(&q->nodes[node].next, relaxed);
atomic_store_explicit(&q->nodes[node].next, tmp, relaxed);
while (!success) {
- //tail isn't actually synchronization point
- tail = atomic_load_explicit(&q->tail, relaxed);
- //acquire reference...
- next = atomic_load_explicit(&q->nodes[get_ptr(tail)].next, memory_order_seq_cst);
- if (tail == atomic_load_explicit(&q->tail, memory_order_seq_cst)) {
+ tail = atomic_load_explicit(&q->tail, acquire);
+ next = atomic_load_explicit(&q->nodes[get_ptr(tail)].next, acquire);
+ if (tail == atomic_load_explicit(&q->tail, relaxed)) {
+
+ /* Check for uninitialized 'next' */
+ MODEL_ASSERT(get_ptr(next) != POISON_IDX);
+
if (get_ptr(next) == 0) { // == NULL
pointer value = MAKE_POINTER(node, get_count(next) + 1);
success = atomic_compare_exchange_strong_explicit(&q->nodes[get_ptr(tail)].next,
- &next, value, memory_order_seq_cst, memory_order_seq_cst);
+ &next, value, release, release);
}
if (!success) {
- unsigned int ptr = get_ptr(atomic_load_explicit(&q->nodes[get_ptr(tail)].next, memory_order_relaxed));
+ unsigned int ptr = get_ptr(atomic_load_explicit(&q->nodes[get_ptr(tail)].next, acquire));
pointer value = MAKE_POINTER(ptr,
get_count(tail) + 1);
atomic_compare_exchange_strong_explicit(&q->tail,
&tail, value,
- memory_order_seq_cst, memory_order_seq_cst);
+ release, release);
+ thrd_yield();
}
}
}
atomic_compare_exchange_strong_explicit(&q->tail,
&tail,
MAKE_POINTER(node, get_count(tail) + 1),
- memory_order_seq_cst, memory_order_seq_cst);
+ release, release);
}
unsigned int dequeue(queue_t *q)
pointer next;
while (!success) {
- head = atomic_load_explicit(&q->head, memory_order_seq_cst);
- tail = atomic_load_explicit(&q->tail, memory_order_seq_cst);
- next = atomic_load_explicit(&q->nodes[get_ptr(head)].next, memory_order_seq_cst);
- if (atomic_load_explicit(&q->head, memory_order_seq_cst) == head) {
+ head = atomic_load_explicit(&q->head, acquire);
+ tail = atomic_load_explicit(&q->tail, relaxed);
+ next = atomic_load_explicit(&q->nodes[get_ptr(head)].next, acquire);
+ if (atomic_load_explicit(&q->head, relaxed) == head) {
if (get_ptr(head) == get_ptr(tail)) {
+
+ /* Check for uninitialized 'next' */
+ MODEL_ASSERT(get_ptr(next) != POISON_IDX);
+
if (get_ptr(next) == 0) { // NULL
return 0; // NULL
}
atomic_compare_exchange_strong_explicit(&q->tail,
&tail,
MAKE_POINTER(get_ptr(next), get_count(tail) + 1),
- memory_order_seq_cst, memory_order_seq_cst);
+ release, release);
+ thrd_yield();
} else {
value = load_32(&q->nodes[get_ptr(next)].value);
success = atomic_compare_exchange_strong_explicit(&q->head,
&head,
MAKE_POINTER(get_ptr(next), get_count(head) + 1),
- memory_order_seq_cst, memory_order_seq_cst);
+ release, release);
+ if (!success)
+ thrd_yield();
}
}
}