sched/walt: Accounting for number of irqs pending on each core

[firefly-linux-kernel-4.4.55.git] / kernel / sched / rt.c

diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 8ec86abe0ea188369ee4e7efd21789d8cb127c14..be700bfa1ae493e8728513fd117ae1bff2da199a 100644 (file)
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -8,6 +8,8 @@
 #include <linux/slab.h>
 #include <linux/irq_work.h>
 
+#include "walt.h"
+
 int sched_rr_timeslice = RR_TIMESLICE;
 
 static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
@@ -1261,6 +1263,7 @@ enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)
                rt_se->timeout = 0;
 
        enqueue_rt_entity(rt_se, flags & ENQUEUE_HEAD);
+       walt_inc_cumulative_runnable_avg(rq, p);
 
        if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
                enqueue_pushable_task(rq, p);
@@ -1272,6 +1275,7 @@ static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags)
 
        update_curr_rt(rq);
        dequeue_rt_entity(rt_se);
+       walt_dec_cumulative_runnable_avg(rq, p);
 
        dequeue_pushable_task(rq, p);
 }
@@ -1426,6 +1430,41 @@ static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int flag
 #endif
 }
 
+#ifdef CONFIG_SMP
+static void sched_rt_update_capacity_req(struct rq *rq)
+{
+       u64 total, used, age_stamp, avg;
+       s64 delta;
+
+       if (!sched_freq())
+               return;
+
+       sched_avg_update(rq);
+       /*
+        * Since we're reading these variables without serialization make sure
+        * we read them once before doing sanity checks on them.
+        */
+       age_stamp = READ_ONCE(rq->age_stamp);
+       avg = READ_ONCE(rq->rt_avg);
+       delta = rq_clock(rq) - age_stamp;
+
+       if (unlikely(delta < 0))
+               delta = 0;
+
+       total = sched_avg_period() + delta;
+
+       used = div_u64(avg, total);
+       if (unlikely(used > SCHED_CAPACITY_SCALE))
+               used = SCHED_CAPACITY_SCALE;
+
+       set_rt_cpu_capacity(rq->cpu, 1, (unsigned long)(used));
+}
+#else
+static inline void sched_rt_update_capacity_req(struct rq *rq)
+{ }
+
+#endif
+
 static struct sched_rt_entity *pick_next_rt_entity(struct rq *rq,
                                                   struct rt_rq *rt_rq)
 {
@@ -1494,8 +1533,17 @@ pick_next_task_rt(struct rq *rq, struct task_struct *prev)
        if (prev->sched_class == &rt_sched_class)
                update_curr_rt(rq);
 
-       if (!rt_rq->rt_queued)
+       if (!rt_rq->rt_queued) {
+               /*
+                * The next task to be picked on this rq will have a lower
+                * priority than rt tasks so we can spend some time to update
+                * the capacity used by rt tasks based on the last activity.
+                * This value will be the used as an estimation of the next
+                * activity.
+                */
+               sched_rt_update_capacity_req(rq);
                return NULL;
+       }
 
        put_prev_task(rq, prev);
 
@@ -2212,6 +2260,9 @@ static void task_tick_rt(struct rq *rq, struct task_struct *p, int queued)
 
        update_curr_rt(rq);
 
+       if (rq->rt.rt_nr_running)
+               sched_rt_update_capacity_req(rq);
+
        watchdog(rq, p);
 
        /*