Merge branch 'timers-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...

[firefly-linux-kernel-4.4.55.git] / kernel / sched / deadline.c

diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 21d6907d2b9fd07c47d9e9c9125d6d5b47f1b499..eac20c557a55cc83f8e9d7e62578868ba9436aff 100644 (file)
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -632,7 +632,7 @@ void init_dl_task_timer(struct sched_dl_entity *dl_se)
 }
 
 static
-int dl_runtime_exceeded(struct rq *rq, struct sched_dl_entity *dl_se)
+int dl_runtime_exceeded(struct sched_dl_entity *dl_se)
 {
        return (dl_se->runtime <= 0);
 }
@@ -676,7 +676,7 @@ static void update_curr_dl(struct rq *rq)
        sched_rt_avg_update(rq, delta_exec);
 
        dl_se->runtime -= dl_se->dl_yielded ? 0 : delta_exec;
-       if (dl_runtime_exceeded(rq, dl_se)) {
+       if (dl_runtime_exceeded(dl_se)) {
                dl_se->dl_throttled = 1;
                __dequeue_task_dl(rq, curr, 0);
                if (unlikely(!start_dl_timer(dl_se, curr->dl.dl_boosted)))
@@ -987,7 +987,7 @@ select_task_rq_dl(struct task_struct *p, int cpu, int sd_flag, int flags)
        rq = cpu_rq(cpu);
 
        rcu_read_lock();
-       curr = ACCESS_ONCE(rq->curr); /* unlocked access */
+       curr = READ_ONCE(rq->curr); /* unlocked access */
 
        /*
         * If we are dealing with a -deadline task, we must
@@ -1004,7 +1004,9 @@ select_task_rq_dl(struct task_struct *p, int cpu, int sd_flag, int flags)
            (p->nr_cpus_allowed > 1)) {
                int target = find_later_rq(p);
 
-               if (target != -1)
+               if (target != -1 &&
+                               dl_time_before(p->dl.deadline,
+                                       cpu_rq(target)->dl.earliest_dl.curr))
                        cpu = target;
        }
        rcu_read_unlock();
@@ -1222,6 +1224,32 @@ next_node:
        return NULL;
 }
 
+/*
+ * Return the earliest pushable rq's task, which is suitable to be executed
+ * on the CPU, NULL otherwise:
+ */
+static struct task_struct *pick_earliest_pushable_dl_task(struct rq *rq, int cpu)
+{
+       struct rb_node *next_node = rq->dl.pushable_dl_tasks_leftmost;
+       struct task_struct *p = NULL;
+
+       if (!has_pushable_dl_tasks(rq))
+               return NULL;
+
+next_node:
+       if (next_node) {
+               p = rb_entry(next_node, struct task_struct, pushable_dl_tasks);
+
+               if (pick_dl_task(rq, p, cpu))
+                       return p;
+
+               next_node = rb_next(next_node);
+               goto next_node;
+       }
+
+       return NULL;
+}
+
 static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask_dl);
 
 static int find_later_rq(struct task_struct *task)
@@ -1325,6 +1353,17 @@ static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq)
 
                later_rq = cpu_rq(cpu);
 
+               if (!dl_time_before(task->dl.deadline,
+                                       later_rq->dl.earliest_dl.curr)) {
+                       /*
+                        * Target rq has tasks of equal or earlier deadline,
+                        * retrying does not release any lock and is unlikely
+                        * to yield a different result.
+                        */
+                       later_rq = NULL;
+                       break;
+               }
+
                /* Retry if something changed. */
                if (double_lock_balance(rq, later_rq)) {
                        if (unlikely(task_rq(task) != rq ||
@@ -1506,7 +1545,7 @@ static int pull_dl_task(struct rq *this_rq)
                if (src_rq->dl.dl_nr_running <= 1)
                        goto skip;
 
-               p = pick_next_earliest_dl_task(src_rq, this_cpu);
+               p = pick_earliest_pushable_dl_task(src_rq, this_cpu);
 
                /*
                 * We found a task to be pulled if:
@@ -1651,7 +1690,7 @@ static void rq_offline_dl(struct rq *rq)
        cpudl_clear_freecpu(&rq->rd->cpudl, rq->cpu);
 }
 
-void init_sched_dl_class(void)
+void __init init_sched_dl_class(void)
 {
        unsigned int i;