*/
#define CFQ_QHASH_SHIFT 6
#define CFQ_QHASH_ENTRIES (1 << CFQ_QHASH_SHIFT)
-#define list_entry_qhash(entry) hlist_entry((entry), struct cfq_queue, cfq_hash)
-
-#define list_entry_cfqq(ptr) list_entry((ptr), struct cfq_queue, cfq_list)
#define RQ_CIC(rq) ((struct cfq_io_context*)(rq)->elevator_private)
#define RQ_CFQQ(rq) ((rq)->elevator_private2)
* rr list of queues with requests and the count of them
*/
struct cfq_rb_root service_tree;
- struct list_head cur_rr;
unsigned int busy_queues;
/*
struct cfq_queue *active_queue;
struct cfq_io_context *active_cic;
- unsigned int dispatch_slice;
struct timer_list idle_class_timer;
struct hlist_node cfq_hash;
/* hash key */
unsigned int key;
- /* member of the rr/busy/cur/idle cfqd list */
- struct list_head cfq_list;
/* service_tree member */
struct rb_node rb_node;
/* service_tree key */
/*
* just an approximation, should be ok.
*/
- return ((cfqd->busy_queues - 1) * cfq_prio_slice(cfqd, 1, 0));
+ return (cfqd->busy_queues - 1) * (cfq_prio_slice(cfqd, 1, 0) -
+ cfq_prio_slice(cfqd, cfq_cfqq_sync(cfqq), cfqq->ioprio));
}
/*
* we will service the queues.
*/
static void cfq_service_tree_add(struct cfq_data *cfqd,
- struct cfq_queue *cfqq)
+ struct cfq_queue *cfqq, int add_front)
{
struct rb_node **p = &cfqd->service_tree.rb.rb_node;
struct rb_node *parent = NULL;
unsigned long rb_key;
int left;
- rb_key = cfq_slice_offset(cfqd, cfqq) + jiffies;
- rb_key += cfqq->slice_resid;
- cfqq->slice_resid = 0;
+ if (!add_front) {
+ rb_key = cfq_slice_offset(cfqd, cfqq) + jiffies;
+ rb_key += cfqq->slice_resid;
+ cfqq->slice_resid = 0;
+ } else
+ rb_key = 0;
if (!RB_EMPTY_NODE(&cfqq->rb_node)) {
/*
/*
* Update cfqq's position in the service tree.
*/
-static void cfq_resort_rr_list(struct cfq_queue *cfqq, int preempted)
+static void cfq_resort_rr_list(struct cfq_data *cfqd, struct cfq_queue *cfqq)
{
/*
* Resorting requires the cfqq to be on the RR list already.
*/
if (cfq_cfqq_on_rr(cfqq))
- cfq_service_tree_add(cfqq->cfqd, cfqq);
+ cfq_service_tree_add(cfqd, cfqq, 0);
}
/*
cfq_mark_cfqq_on_rr(cfqq);
cfqd->busy_queues++;
- cfq_resort_rr_list(cfqq, 0);
+ cfq_resort_rr_list(cfqd, cfqq);
}
/*
{
BUG_ON(!cfq_cfqq_on_rr(cfqq));
cfq_clear_cfqq_on_rr(cfqq);
- list_del_init(&cfqq->cfq_list);
if (!RB_EMPTY_NODE(&cfqq->rb_node))
cfq_rb_erase(&cfqq->rb_node, &cfqd->service_tree);
*/
static void
__cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq,
- int preempted, int timed_out)
+ int timed_out)
{
if (cfq_cfqq_wait_request(cfqq))
del_timer(&cfqd->idle_slice_timer);
cfq_clear_cfqq_wait_request(cfqq);
/*
- * store what was left of this slice, if the queue idled out
- * or was preempted
+ * store what was left of this slice, if the queue idled/timed out
*/
if (timed_out && !cfq_cfqq_slice_new(cfqq))
cfqq->slice_resid = cfqq->slice_end - jiffies;
- cfq_resort_rr_list(cfqq, preempted);
+ cfq_resort_rr_list(cfqd, cfqq);
if (cfqq == cfqd->active_queue)
cfqd->active_queue = NULL;
put_io_context(cfqd->active_cic->ioc);
cfqd->active_cic = NULL;
}
-
- cfqd->dispatch_slice = 0;
}
-static inline void cfq_slice_expired(struct cfq_data *cfqd, int preempted,
- int timed_out)
+static inline void cfq_slice_expired(struct cfq_data *cfqd, int timed_out)
{
struct cfq_queue *cfqq = cfqd->active_queue;
if (cfqq)
- __cfq_slice_expired(cfqd, cfqq, preempted, timed_out);
+ __cfq_slice_expired(cfqd, cfqq, timed_out);
}
/*
*/
static struct cfq_queue *cfq_get_next_queue(struct cfq_data *cfqd)
{
- struct cfq_queue *cfqq = NULL;
+ struct cfq_queue *cfqq;
+ struct rb_node *n;
- if (!list_empty(&cfqd->cur_rr)) {
- /*
- * if current list is non-empty, grab first entry.
- */
- cfqq = list_entry_cfqq(cfqd->cur_rr.next);
- } else if (!RB_EMPTY_ROOT(&cfqd->service_tree.rb)) {
- struct rb_node *n = cfq_rb_first(&cfqd->service_tree);
+ if (RB_EMPTY_ROOT(&cfqd->service_tree.rb))
+ return NULL;
- cfqq = rb_entry(n, struct cfq_queue, rb_node);
- if (cfq_class_idle(cfqq)) {
- unsigned long end;
+ n = cfq_rb_first(&cfqd->service_tree);
+ cfqq = rb_entry(n, struct cfq_queue, rb_node);
- /*
- * if we have idle queues and no rt or be queues had
- * pending requests, either allow immediate service if
- * the grace period has passed or arm the idle grace
- * timer
- */
- end = cfqd->last_end_request + CFQ_IDLE_GRACE;
- if (time_before(jiffies, end)) {
- mod_timer(&cfqd->idle_class_timer, end);
- cfqq = NULL;
- }
+ if (cfq_class_idle(cfqq)) {
+ unsigned long end;
+
+ /*
+ * if we have idle queues and no rt or be queues had
+ * pending requests, either allow immediate service if
+ * the grace period has passed or arm the idle grace
+ * timer
+ */
+ end = cfqd->last_end_request + CFQ_IDLE_GRACE;
+ if (time_before(jiffies, end)) {
+ mod_timer(&cfqd->idle_class_timer, end);
+ cfqq = NULL;
}
}
}
expire:
- cfq_slice_expired(cfqd, 0, 0);
+ cfq_slice_expired(cfqd, 0);
new_queue:
cfqq = cfq_set_active_queue(cfqd);
keep_queue:
*/
cfq_dispatch_insert(cfqd->queue, rq);
- cfqd->dispatch_slice++;
dispatched++;
if (!cfqd->active_cic) {
* queue always expire after 1 dispatch round.
*/
if (cfqd->busy_queues > 1 && ((!cfq_cfqq_sync(cfqq) &&
- cfqd->dispatch_slice >= cfq_prio_to_maxrq(cfqd, cfqq)) ||
+ dispatched >= cfq_prio_to_maxrq(cfqd, cfqq)) ||
cfq_class_idle(cfqq))) {
cfqq->slice_end = jiffies + 1;
- cfq_slice_expired(cfqd, 0, 0);
+ cfq_slice_expired(cfqd, 0);
}
return dispatched;
return dispatched;
}
-static int cfq_forced_dispatch_cfqqs(struct list_head *list)
-{
- struct cfq_queue *cfqq, *next;
- int dispatched;
-
- dispatched = 0;
- list_for_each_entry_safe(cfqq, next, list, cfq_list)
- dispatched += __cfq_forced_dispatch_cfqq(cfqq);
-
- return dispatched;
-}
-
/*
* Drain our current requests. Used for barriers and when switching
* io schedulers on-the-fly.
dispatched += __cfq_forced_dispatch_cfqq(cfqq);
}
- dispatched += cfq_forced_dispatch_cfqqs(&cfqd->cur_rr);
-
- cfq_slice_expired(cfqd, 0, 0);
+ cfq_slice_expired(cfqd, 0);
BUG_ON(cfqd->busy_queues);
BUG_ON(cfq_cfqq_on_rr(cfqq));
if (unlikely(cfqd->active_queue == cfqq)) {
- __cfq_slice_expired(cfqd, cfqq, 0, 0);
+ __cfq_slice_expired(cfqd, cfqq, 0);
cfq_schedule_dispatch(cfqd);
}
static void cfq_exit_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq)
{
if (unlikely(cfqq == cfqd->active_queue)) {
- __cfq_slice_expired(cfqd, cfqq, 0, 0);
+ __cfq_slice_expired(cfqd, cfqq, 0);
cfq_schedule_dispatch(cfqd);
}
memset(cfqq, 0, sizeof(*cfqq));
INIT_HLIST_NODE(&cfqq->cfq_hash);
- INIT_LIST_HEAD(&cfqq->cfq_list);
RB_CLEAR_NODE(&cfqq->rb_node);
INIT_LIST_HEAD(&cfqq->fifo);
*/
static void cfq_preempt_queue(struct cfq_data *cfqd, struct cfq_queue *cfqq)
{
- cfq_slice_expired(cfqd, 1, 1);
+ cfq_slice_expired(cfqd, 1);
/*
* Put the new queue at the front of the of the current list,
* so we know that it will be selected next.
*/
BUG_ON(!cfq_cfqq_on_rr(cfqq));
- list_del_init(&cfqq->cfq_list);
- list_add(&cfqq->cfq_list, &cfqd->cur_rr);
+
+ cfq_service_tree_add(cfqd, cfqq, 1);
cfqq->slice_end = 0;
cfq_mark_cfqq_slice_new(cfqq);
cfq_clear_cfqq_slice_new(cfqq);
}
if (cfq_slice_used(cfqq))
- cfq_slice_expired(cfqd, 0, 1);
+ cfq_slice_expired(cfqd, 1);
else if (sync && RB_EMPTY_ROOT(&cfqq->sort_list))
cfq_arm_slice_timer(cfqd);
}
}
}
expire:
- cfq_slice_expired(cfqd, 0, timed_out);
+ cfq_slice_expired(cfqd, timed_out);
out_kick:
cfq_schedule_dispatch(cfqd);
out_cont:
spin_lock_irq(q->queue_lock);
if (cfqd->active_queue)
- __cfq_slice_expired(cfqd, cfqd->active_queue, 0, 0);
+ __cfq_slice_expired(cfqd, cfqd->active_queue, 0);
while (!list_empty(&cfqd->cic_list)) {
struct cfq_io_context *cic = list_entry(cfqd->cic_list.next,
memset(cfqd, 0, sizeof(*cfqd));
cfqd->service_tree = CFQ_RB_ROOT;
- INIT_LIST_HEAD(&cfqd->cur_rr);
INIT_LIST_HEAD(&cfqd->cic_list);
cfqd->cfq_hash = kmalloc_node(sizeof(struct hlist_head) * CFQ_QHASH_ENTRIES, GFP_KERNEL, q->node);
projects / firefly-linux-kernel-4.4.55.git / blobdiff