sched: Consider misfit tasks when load-balancing

[firefly-linux-kernel-4.4.55.git] / kernel / sched / fair.c

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index b16e1b7878f6c9e6c1b3af41eac2831e3f0711c1..5b8a9c68f3fb70d9d7f56097d619b654a7505b98 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -6016,6 +6016,7 @@ struct lb_env {
        unsigned int            loop_max;
 
        enum fbq_type           fbq_type;
+       enum group_type         busiest_group_type;
        struct list_head        tasks;
 };
 
@@ -6780,6 +6781,18 @@ group_is_overloaded(struct lb_env *env, struct sg_lb_stats *sgs)
        return false;
 }
 
+
+/*
+ * group_smaller_cpu_capacity: Returns true if sched_group sg has smaller
+ * per-cpu capacity than sched_group ref.
+ */
+static inline bool
+group_smaller_cpu_capacity(struct sched_group *sg, struct sched_group *ref)
+{
+       return sg->sgc->max_capacity + capacity_margin - SCHED_LOAD_SCALE <
+                                                       ref->sgc->max_capacity;
+}
+
 static inline enum
 group_type group_classify(struct sched_group *group,
                          struct sg_lb_stats *sgs)
@@ -6886,9 +6899,25 @@ static bool update_sd_pick_busiest(struct lb_env *env,
        if (sgs->group_type < busiest->group_type)
                return false;
 
+       /*
+        * Candidate sg doesn't face any serious load-balance problems
+        * so don't pick it if the local sg is already filled up.
+        */
+       if (sgs->group_type == group_other &&
+           !group_has_capacity(env, &sds->local_stat))
+               return false;
+
        if (sgs->avg_load <= busiest->avg_load)
                return false;
 
+       /*
+        * Candiate sg has no more than one task per cpu and has higher
+        * per-cpu capacity. No reason to pull tasks to less capable cpus.
+        */
+       if (sgs->sum_nr_running <= sgs->group_weight &&
+           group_smaller_cpu_capacity(sds->local, sg))
+               return false;
+
        /* This is the busiest node in its class. */
        if (!(env->sd->flags & SD_ASYM_PACKING))
                return true;
@@ -6994,6 +7023,15 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd
                        sgs->group_type = group_classify(sg, sgs);
                }
 
+               /*
+                * Ignore task groups with misfit tasks if local group has no
+                * capacity or if per-cpu capacity isn't higher.
+                */
+               if (sgs->group_type == group_misfit_task &&
+                   (!group_has_capacity(env, &sds->local_stat) ||
+                    !group_smaller_cpu_capacity(sg, sds->local)))
+                       sgs->group_type = group_other;
+
                if (update_sd_pick_busiest(env, sds, sg, sgs)) {
                        sds->busiest = sg;
                        sds->busiest_stat = *sgs;
@@ -7170,6 +7208,22 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s
         */
        if (busiest->avg_load <= sds->avg_load ||
            local->avg_load >= sds->avg_load) {
+               /* Misfitting tasks should be migrated in any case */
+               if (busiest->group_type == group_misfit_task) {
+                       env->imbalance = busiest->group_misfit_task;
+                       return;
+               }
+
+               /*
+                * Busiest group is overloaded, local is not, use the spare
+                * cycles to maximize throughput
+                */
+               if (busiest->group_type == group_overloaded &&
+                   local->group_type <= group_misfit_task) {
+                       env->imbalance = busiest->load_per_task;
+                       return;
+               }
+
                env->imbalance = 0;
                return fix_small_imbalance(env, sds);
        }
@@ -7203,6 +7257,11 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s
                (sds->avg_load - local->avg_load) * local->group_capacity
        ) / SCHED_CAPACITY_SCALE;
 
+       /* Boost imbalance to allow misfit task to be balanced. */
+       if (busiest->group_type == group_misfit_task)
+               env->imbalance = max_t(long, env->imbalance,
+                                    busiest->group_misfit_task);
+
        /*
         * if *imbalance is less than the average load per runnable task
         * there is no guarantee that any tasks will be moved so we'll have
@@ -7276,6 +7335,11 @@ static struct sched_group *find_busiest_group(struct lb_env *env)
            busiest->group_no_capacity)
                goto force_balance;
 
+       /* Misfitting tasks should be dealt with regardless of the avg load */
+       if (busiest->group_type == group_misfit_task) {
+               goto force_balance;
+       }
+
        /*
         * If the local group is busier than the selected busiest group
         * don't try and pull any tasks.
@@ -7299,7 +7363,8 @@ static struct sched_group *find_busiest_group(struct lb_env *env)
                 * might end up to just move the imbalance on another group
                 */
                if ((busiest->group_type != group_overloaded) &&
-                               (local->idle_cpus <= (busiest->idle_cpus + 1)))
+                   (local->idle_cpus <= (busiest->idle_cpus + 1)) &&
+                   !group_smaller_cpu_capacity(sds.busiest, sds.local))
                        goto out_balanced;
        } else {
                /*
@@ -7312,6 +7377,7 @@ static struct sched_group *find_busiest_group(struct lb_env *env)
        }
 
 force_balance:
+       env->busiest_group_type = busiest->group_type;
        /* Looks like there is an imbalance. Compute it */
        calculate_imbalance(env, &sds);
        return sds.busiest;
@@ -7370,7 +7436,8 @@ static struct rq *find_busiest_queue(struct lb_env *env,
                 */
 
                if (rq->nr_running == 1 && wl > env->imbalance &&
-                   !check_cpu_capacity(rq, env->sd))
+                   !check_cpu_capacity(rq, env->sd) &&
+                   env->busiest_group_type != group_misfit_task)
                        continue;
 
                /*