sched: scheduler-driven cpu frequency selection

[firefly-linux-kernel-4.4.55.git] / kernel / sched / sched.h

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index efd3bfc7e34722883e2f08ca82f91cffde812963..ac28024b2d7a4129bef689880994e793ce2070a9 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -510,6 +510,12 @@ struct dl_rq {
 
 #ifdef CONFIG_SMP
 
+struct max_cpu_capacity {
+       raw_spinlock_t lock;
+       unsigned long val;
+       int cpu;
+};
+
 /*
  * We add the notion of a root-domain which will be used to define per-domain
  * variables. Each exclusive cpuset essentially defines an island domain by
@@ -528,6 +534,9 @@ struct root_domain {
        /* Indicate more than one runnable task for any CPU */
        bool overload;
 
+       /* Indicate one or more cpus over-utilized (tipping point) */
+       bool overutilized;
+
        /*
         * The bit corresponding to a CPU gets set here if such CPU has more
         * than one runnable -deadline task (as it is below for RT tasks).
@@ -543,6 +552,9 @@ struct root_domain {
         */
        cpumask_var_t rto_mask;
        struct cpupri cpupri;
+
+       /* Maximum cpu capacity in the system. */
+       struct max_cpu_capacity max_cpu_capacity;
 };
 
 extern struct root_domain def_root_domain;
@@ -572,6 +584,7 @@ struct rq {
        #define CPU_LOAD_IDX_MAX 5
        unsigned long cpu_load[CPU_LOAD_IDX_MAX];
        unsigned long last_load_update_tick;
+       unsigned int misfit_task;
 #ifdef CONFIG_NO_HZ_COMMON
        u64 nohz_stamp;
        unsigned long nohz_flags;
@@ -687,6 +700,7 @@ struct rq {
 #ifdef CONFIG_CPU_IDLE
        /* Must be inspected within a rcu lock section */
        struct cpuidle_state *idle_state;
+       int idle_state_idx;
 #endif
 };
 
@@ -836,6 +850,8 @@ DECLARE_PER_CPU(int, sd_llc_id);
 DECLARE_PER_CPU(struct sched_domain *, sd_numa);
 DECLARE_PER_CPU(struct sched_domain *, sd_busy);
 DECLARE_PER_CPU(struct sched_domain *, sd_asym);
+DECLARE_PER_CPU(struct sched_domain *, sd_ea);
+DECLARE_PER_CPU(struct sched_domain *, sd_scs);
 
 struct sched_group_capacity {
        atomic_t ref;
@@ -843,7 +859,8 @@ struct sched_group_capacity {
         * CPU capacity of this group, SCHED_LOAD_SCALE being max capacity
         * for a single CPU.
         */
-       unsigned int capacity;
+       unsigned long capacity;
+       unsigned long max_capacity; /* Max per-cpu capacity in group */
        unsigned long next_update;
        int imbalance; /* XXX unrelated to capacity but shared group state */
        /*
@@ -860,6 +877,7 @@ struct sched_group {
 
        unsigned int group_weight;
        struct sched_group_capacity *sgc;
+       const struct sched_group_energy const *sge;
 
        /*
         * The CPUs this group covers.
@@ -1073,6 +1091,9 @@ static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
         * We must ensure this doesn't happen until the switch is completely
         * finished.
         *
+        * In particular, the load of prev->state in finish_task_switch() must
+        * happen before this.
+        *
         * Pairs with the control dependency and rmb in try_to_wake_up().
         */
        smp_store_release(&prev->on_cpu, 0);
@@ -1273,6 +1294,17 @@ static inline struct cpuidle_state *idle_get_state(struct rq *rq)
        WARN_ON(!rcu_read_lock_held());
        return rq->idle_state;
 }
+
+static inline void idle_set_state_idx(struct rq *rq, int idle_state_idx)
+{
+       rq->idle_state_idx = idle_state_idx;
+}
+
+static inline int idle_get_state_idx(struct rq *rq)
+{
+       WARN_ON(!rcu_read_lock_held());
+       return rq->idle_state_idx;
+}
 #else
 static inline void idle_set_state(struct rq *rq,
                                  struct cpuidle_state *idle_state)
@@ -1283,6 +1315,15 @@ static inline struct cpuidle_state *idle_get_state(struct rq *rq)
 {
        return NULL;
 }
+
+static inline void idle_set_state_idx(struct rq *rq, int idle_state_idx)
+{
+}
+
+static inline int idle_get_state_idx(struct rq *rq)
+{
+       return -1;
+}
 #endif
 
 extern void sysrq_sched_debug_show(void);
@@ -1307,6 +1348,8 @@ unsigned long to_ratio(u64 period, u64 runtime);
 
 extern void init_entity_runnable_average(struct sched_entity *se);
 
+extern void init_max_cpu_capacity(struct max_cpu_capacity *mcc);
+
 static inline void add_nr_running(struct rq *rq, unsigned count)
 {
        unsigned prev_nr = rq->nr_running;
@@ -1412,6 +1455,57 @@ unsigned long arch_scale_cpu_capacity(struct sched_domain *sd, int cpu)
 }
 #endif
 
+#ifdef CONFIG_CPU_FREQ_GOV_SCHED
+extern unsigned int capacity_margin;
+extern struct static_key __sched_freq;
+
+static inline bool sched_freq(void)
+{
+       return static_key_false(&__sched_freq);
+}
+
+DECLARE_PER_CPU(struct sched_capacity_reqs, cpu_sched_capacity_reqs);
+void update_cpu_capacity_request(int cpu, bool request);
+
+static inline void set_cfs_cpu_capacity(int cpu, bool request,
+                                       unsigned long capacity)
+{
+       if (per_cpu(cpu_sched_capacity_reqs, cpu).cfs != capacity) {
+               per_cpu(cpu_sched_capacity_reqs, cpu).cfs = capacity;
+               update_cpu_capacity_request(cpu, request);
+       }
+}
+
+static inline void set_rt_cpu_capacity(int cpu, bool request,
+                                      unsigned long capacity)
+{
+       if (per_cpu(cpu_sched_capacity_reqs, cpu).rt != capacity) {
+               per_cpu(cpu_sched_capacity_reqs, cpu).rt = capacity;
+               update_cpu_capacity_request(cpu, request);
+       }
+}
+
+static inline void set_dl_cpu_capacity(int cpu, bool request,
+                                      unsigned long capacity)
+{
+       if (per_cpu(cpu_sched_capacity_reqs, cpu).dl != capacity) {
+               per_cpu(cpu_sched_capacity_reqs, cpu).dl = capacity;
+               update_cpu_capacity_request(cpu, request);
+       }
+}
+#else
+static inline bool sched_freq(void) { return false; }
+static inline void set_cfs_cpu_capacity(int cpu, bool request,
+                                       unsigned long capacity)
+{ }
+static inline void set_rt_cpu_capacity(int cpu, bool request,
+                                      unsigned long capacity)
+{ }
+static inline void set_dl_cpu_capacity(int cpu, bool request,
+                                      unsigned long capacity)
+{ }
+#endif
+
 static inline void sched_rt_avg_update(struct rq *rq, u64 rt_delta)
 {
        rq->rt_avg += rt_delta * arch_scale_freq_capacity(NULL, cpu_of(rq));