Merge branch 'linux-linaro-lsk-v4.4-android' of git://git.linaro.org/kernel/linux...

[firefly-linux-kernel-4.4.55.git] / kernel / sched / sched.h

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index a537f1864dd08bad9b94563eeac44912da276f09..2f2b959ad24485ca8eaf2907b2fc66d58e04bc33 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -410,6 +410,10 @@ struct cfs_rq {
        struct list_head leaf_cfs_rq_list;
        struct task_group *tg;  /* group that "owns" this runqueue */
 
+#ifdef CONFIG_SCHED_WALT
+       u64 cumulative_runnable_avg;
+#endif
+
 #ifdef CONFIG_CFS_BANDWIDTH
        int runtime_enabled;
        u64 runtime_expires;
@@ -594,6 +598,14 @@ struct rq {
 #ifdef CONFIG_NO_HZ_FULL
        unsigned long last_sched_tick;
 #endif
+
+#ifdef CONFIG_CPU_QUIET
+       /* time-based average load */
+       u64 nr_last_stamp;
+       u64 nr_running_integral;
+       seqcount_t ave_seqcnt;
+#endif
+
        /* capture load from *all* tasks on this cpu: */
        struct load_weight load;
        unsigned long nr_load_updates;
@@ -655,6 +667,30 @@ struct rq {
        u64 max_idle_balance_cost;
 #endif
 
+#ifdef CONFIG_SCHED_WALT
+       /*
+        * max_freq = user or thermal defined maximum
+        * max_possible_freq = maximum supported by hardware
+        */
+       unsigned int cur_freq, max_freq, min_freq, max_possible_freq;
+       struct cpumask freq_domain_cpumask;
+
+       u64 cumulative_runnable_avg;
+       int efficiency; /* Differentiate cpus with different IPC capability */
+       int load_scale_factor;
+       int capacity;
+       int max_possible_capacity;
+       u64 window_start;
+       u64 curr_runnable_sum;
+       u64 prev_runnable_sum;
+       u64 nt_curr_runnable_sum;
+       u64 nt_prev_runnable_sum;
+       u64 cur_irqload;
+       u64 avg_irqload;
+       u64 irqload_ts;
+#endif /* CONFIG_SCHED_WALT */
+
+
 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
        u64 prev_irq_time;
 #endif
@@ -1269,6 +1305,7 @@ extern const struct sched_class idle_sched_class;
 
 #ifdef CONFIG_SMP
 
+extern void init_max_cpu_capacity(struct max_cpu_capacity *mcc);
 extern void update_group_capacity(struct sched_domain *sd, int cpu);
 
 extern void trigger_load_balance(struct rq *rq);
@@ -1351,9 +1388,7 @@ unsigned long to_ratio(u64 period, u64 runtime);
 
 extern void init_entity_runnable_average(struct sched_entity *se);
 
-extern void init_max_cpu_capacity(struct max_cpu_capacity *mcc);
-
-static inline void add_nr_running(struct rq *rq, unsigned count)
+static inline void __add_nr_running(struct rq *rq, unsigned count)
 {
        unsigned prev_nr = rq->nr_running;
 
@@ -1381,11 +1416,48 @@ static inline void add_nr_running(struct rq *rq, unsigned count)
        }
 }
 
-static inline void sub_nr_running(struct rq *rq, unsigned count)
+static inline void __sub_nr_running(struct rq *rq, unsigned count)
 {
        rq->nr_running -= count;
 }
 
+#ifdef CONFIG_CPU_QUIET
+#define NR_AVE_SCALE(x)                ((x) << FSHIFT)
+static inline u64 do_nr_running_integral(struct rq *rq)
+{
+       s64 nr, deltax;
+       u64 nr_running_integral = rq->nr_running_integral;
+
+       deltax = rq->clock_task - rq->nr_last_stamp;
+       nr = NR_AVE_SCALE(rq->nr_running);
+
+       nr_running_integral += nr * deltax;
+
+       return nr_running_integral;
+}
+
+static inline void add_nr_running(struct rq *rq, unsigned count)
+{
+       write_seqcount_begin(&rq->ave_seqcnt);
+       rq->nr_running_integral = do_nr_running_integral(rq);
+       rq->nr_last_stamp = rq->clock_task;
+       __add_nr_running(rq, count);
+       write_seqcount_end(&rq->ave_seqcnt);
+}
+
+static inline void sub_nr_running(struct rq *rq, unsigned count)
+{
+       write_seqcount_begin(&rq->ave_seqcnt);
+       rq->nr_running_integral = do_nr_running_integral(rq);
+       rq->nr_last_stamp = rq->clock_task;
+       __sub_nr_running(rq, count);
+       write_seqcount_end(&rq->ave_seqcnt);
+}
+#else
+#define add_nr_running __add_nr_running
+#define sub_nr_running __sub_nr_running
+#endif
+
 static inline void rq_last_tick_reset(struct rq *rq)
 {
 #ifdef CONFIG_NO_HZ_FULL
@@ -1469,6 +1541,10 @@ static inline unsigned long capacity_orig_of(int cpu)
        return cpu_rq(cpu)->cpu_capacity_orig;
 }
 
+extern unsigned int sysctl_sched_use_walt_cpu_util;
+extern unsigned int walt_ravg_window;
+extern unsigned int walt_disabled;
+
 /*
  * cpu_util returns the amount of capacity of a CPU that is used by CFS
  * tasks. The unit of the return value must be the one of capacity so we can
@@ -1500,6 +1576,12 @@ static inline unsigned long __cpu_util(int cpu, int delta)
        unsigned long util = cpu_rq(cpu)->cfs.avg.util_avg;
        unsigned long capacity = capacity_orig_of(cpu);
 
+#ifdef CONFIG_SCHED_WALT
+       if (!walt_disabled && sysctl_sched_use_walt_cpu_util) {
+               util = cpu_rq(cpu)->prev_runnable_sum << SCHED_LOAD_SHIFT;
+               do_div(util, walt_ravg_window);
+       }
+#endif
        delta += util;
        if (delta < 0)
                return 0;
@@ -1530,8 +1612,27 @@ void update_cpu_capacity_request(int cpu, bool request);
 static inline void set_cfs_cpu_capacity(int cpu, bool request,
                                        unsigned long capacity)
 {
-       if (per_cpu(cpu_sched_capacity_reqs, cpu).cfs != capacity) {
-               per_cpu(cpu_sched_capacity_reqs, cpu).cfs = capacity;
+       struct sched_capacity_reqs *scr = &per_cpu(cpu_sched_capacity_reqs, cpu);
+
+#ifdef CONFIG_SCHED_WALT
+       if (!walt_disabled && sysctl_sched_use_walt_cpu_util) {
+               int rtdl = scr->rt + scr->dl;
+               /*
+                * WALT tracks the utilization of a CPU considering the load
+                * generated by all the scheduling classes.
+                * Since the following call to:
+                *    update_cpu_capacity
+                * is already adding the RT and DL utilizations let's remove
+                * these contributions from the WALT signal.
+                */
+               if (capacity > rtdl)
+                       capacity -= rtdl;
+               else
+                       capacity = 0;
+       }
+#endif
+       if (scr->cfs != capacity) {
+               scr->cfs = capacity;
                update_cpu_capacity_request(cpu, request);
        }
 }
@@ -1657,6 +1758,9 @@ task_rq_unlock(struct rq *rq, struct task_struct *p, unsigned long *flags)
        raw_spin_unlock_irqrestore(&p->pi_lock, *flags);
 }
 
+extern struct rq *lock_rq_of(struct task_struct *p, unsigned long *flags);
+extern void unlock_rq_of(struct rq *rq, struct task_struct *p, unsigned long *flags);
+
 #ifdef CONFIG_SMP
 #ifdef CONFIG_PREEMPT
 
@@ -1729,7 +1833,8 @@ static inline int double_lock_balance(struct rq *this_rq, struct rq *busiest)
 static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
        __releases(busiest->lock)
 {
-       raw_spin_unlock(&busiest->lock);
+       if (this_rq != busiest)
+               raw_spin_unlock(&busiest->lock);
        lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
 }