2 * Copyright (c) 2012-2014 NVIDIA CORPORATION. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; version 2 of the License.
8 * This program is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * You should have received a copy of the GNU General Public License along
14 * with this program; if not, write to the Free Software Foundation, Inc.,
15 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 #include <linux/kernel.h>
20 #include <linux/cpuquiet.h>
21 #include <linux/cpumask.h>
22 #include <linux/module.h>
23 #include <linux/pm_qos.h>
24 #include <linux/jiffies.h>
25 #include <linux/slab.h>
26 #include <linux/cpu.h>
27 #include <linux/sched.h>
35 static struct work_struct runnables_work;
36 static struct kobject *runnables_kobject;
37 static struct timer_list runnables_timer;
39 static RUNNABLES_STATE runnables_state;
40 /* configurable parameters */
41 static unsigned int sample_rate = 20; /* msec */
43 #define NR_FSHIFT_EXP 3
44 #define NR_FSHIFT (1 << NR_FSHIFT_EXP)
45 /* avg run threads * 8 (e.g., 11 = 1.375 threads) */
46 static unsigned int default_thresholds[] = {
50 static unsigned int nr_run_last;
51 static unsigned int nr_run_hysteresis = 2; /* 1 / 2 thread */
52 static unsigned int default_threshold_level = 4; /* 1 / 4 thread */
53 static unsigned int nr_run_thresholds[NR_CPUS];
55 DEFINE_MUTEX(runnables_lock);
57 struct runnables_avg_sample {
58 u64 previous_integral;
60 bool integral_sampled;
64 static DEFINE_PER_CPU(struct runnables_avg_sample, avg_nr_sample);
66 /* EXP = alpha in the exponential moving average.
67 * Alpha = e ^ (-sample_rate / window_size) * FIXED_1
68 * Calculated for sample_rate of 20ms, window size of 100ms
72 static unsigned int get_avg_nr_runnables(void)
74 unsigned int i, sum = 0;
75 static unsigned int avg;
76 struct runnables_avg_sample *sample;
77 u64 integral, old_integral, delta_integral, delta_time, cur_time;
79 for_each_online_cpu(i) {
80 sample = &per_cpu(avg_nr_sample, i);
81 integral = nr_running_integral(i);
82 old_integral = sample->previous_integral;
83 sample->previous_integral = integral;
84 cur_time = ktime_to_ns(ktime_get());
85 delta_time = cur_time - sample->prev_timestamp;
86 sample->prev_timestamp = cur_time;
88 if (!sample->integral_sampled) {
89 sample->integral_sampled = true;
90 /* First sample to initialize prev_integral, skip
96 if (integral < old_integral) {
98 delta_integral = (ULLONG_MAX - old_integral) + integral;
100 delta_integral = integral - old_integral;
103 /* Calculate average for the previous sample window */
104 do_div(delta_integral, delta_time);
105 sample->avg = delta_integral;
109 /* Exponential moving average
110 * Avgn = Avgn-1 * alpha + new_avg * (1 - alpha)
113 avg += sum * (FIXED_1 - EXP);
119 static int get_action(unsigned int nr_run)
121 unsigned int nr_cpus = num_online_cpus();
122 int max_cpus = pm_qos_request(PM_QOS_MAX_ONLINE_CPUS) ? : 4;
123 int min_cpus = pm_qos_request(PM_QOS_MIN_ONLINE_CPUS);
125 if ((nr_cpus > max_cpus || nr_run < nr_cpus) && nr_cpus >= min_cpus)
128 if (nr_cpus < min_cpus || nr_run > nr_cpus)
134 static void runnables_avg_sampler(unsigned long data)
136 unsigned int nr_run, avg_nr_run;
140 if (runnables_state != RUNNING)
143 avg_nr_run = get_avg_nr_runnables();
144 mod_timer(&runnables_timer, jiffies + msecs_to_jiffies(sample_rate));
146 for (nr_run = 1; nr_run < ARRAY_SIZE(nr_run_thresholds); nr_run++) {
147 unsigned int nr_threshold = nr_run_thresholds[nr_run - 1];
148 if (nr_run_last <= nr_run)
149 nr_threshold += NR_FSHIFT / nr_run_hysteresis;
150 if (avg_nr_run <= (nr_threshold << (FSHIFT - NR_FSHIFT_EXP)))
154 nr_run_last = nr_run;
156 action = get_action(nr_run);
159 schedule_work(&runnables_work);
163 static unsigned int get_lightest_loaded_cpu_n(void)
165 unsigned long min_avg_runnables = ULONG_MAX;
166 unsigned int cpu = nr_cpu_ids;
169 for_each_online_cpu(i) {
170 struct runnables_avg_sample *s = &per_cpu(avg_nr_sample, i);
171 unsigned int nr_runnables = s->avg;
172 if (i > 0 && min_avg_runnables > nr_runnables) {
174 min_avg_runnables = nr_runnables;
181 static void runnables_work_func(struct work_struct *work)
183 unsigned int cpu = nr_cpu_ids;
186 if (runnables_state != RUNNING)
189 action = get_action(nr_run_last);
191 cpu = cpumask_next_zero(0, cpu_online_mask);
192 if (cpu < nr_cpu_ids)
193 cpuquiet_wake_cpu(cpu, false);
194 } else if (action < 0) {
195 cpu = get_lightest_loaded_cpu_n();
196 if (cpu < nr_cpu_ids)
197 cpuquiet_quiesence_cpu(cpu, false);
201 #define MAX_BYTES 100
203 static ssize_t show_thresholds(struct cpuquiet_attribute *attr, char *buf)
205 char buffer[MAX_BYTES];
209 for_each_possible_cpu(i) {
210 if (i == ARRAY_SIZE(nr_run_thresholds) - 1)
212 if (size >= sizeof(buffer))
214 size += snprintf(buffer + size, sizeof(buffer) - size,
215 "%u->%u core threshold: %u\n",
216 i + 1, i + 2, nr_run_thresholds[i]);
218 return snprintf(buf, sizeof(buffer), "%s", buffer);
221 static ssize_t store_thresholds(struct cpuquiet_attribute *attr,
222 const char *buf, size_t count)
225 char *val, *str, input[MAX_BYTES];
226 unsigned int thresholds[NR_CPUS];
228 if (!count || count >= MAX_BYTES)
230 strncpy(input, buf, count);
233 memcpy(thresholds, nr_run_thresholds, sizeof(nr_run_thresholds));
234 while ((val = strsep(&str, " ")) != NULL) {
237 if (i == ARRAY_SIZE(nr_run_thresholds) - 1)
239 ret = kstrtouint(val, 10, &thresholds[i]);
245 memcpy(nr_run_thresholds, thresholds, sizeof(thresholds));
249 CPQ_BASIC_ATTRIBUTE(sample_rate, 0644, uint);
250 CPQ_BASIC_ATTRIBUTE(nr_run_hysteresis, 0644, uint);
251 CPQ_ATTRIBUTE_CUSTOM(nr_run_thresholds, 0644,
252 show_thresholds, store_thresholds);
254 static struct attribute *runnables_attributes[] = {
255 &sample_rate_attr.attr,
256 &nr_run_hysteresis_attr.attr,
257 &nr_run_thresholds_attr.attr,
261 static const struct sysfs_ops runnables_sysfs_ops = {
262 .show = cpuquiet_auto_sysfs_show,
263 .store = cpuquiet_auto_sysfs_store,
266 static struct kobj_type ktype_runnables = {
267 .sysfs_ops = &runnables_sysfs_ops,
268 .default_attrs = runnables_attributes,
271 static int runnables_sysfs(void)
275 runnables_kobject = kzalloc(sizeof(*runnables_kobject),
278 if (!runnables_kobject)
281 err = cpuquiet_kobject_init(runnables_kobject, &ktype_runnables,
285 kfree(runnables_kobject);
290 static void runnables_device_busy(void)
292 mutex_lock(&runnables_lock);
293 if (runnables_state == RUNNING) {
294 runnables_state = IDLE;
295 del_timer_sync(&runnables_timer);
297 mutex_unlock(&runnables_lock);
300 static void runnables_device_free(void)
302 mutex_lock(&runnables_lock);
303 if (runnables_state == IDLE) {
304 runnables_state = RUNNING;
305 mod_timer(&runnables_timer, jiffies + 1);
307 mutex_unlock(&runnables_lock);
310 static void runnables_stop(void)
312 mutex_lock(&runnables_lock);
314 runnables_state = DISABLED;
315 del_timer_sync(&runnables_timer);
316 kobject_put(runnables_kobject);
317 kfree(runnables_kobject);
319 mutex_unlock(&runnables_lock);
322 static int runnables_start(void)
326 err = runnables_sysfs();
330 INIT_WORK(&runnables_work, runnables_work_func);
332 init_timer(&runnables_timer);
333 runnables_timer.function = runnables_avg_sampler;
335 for(i = 0; i < ARRAY_SIZE(nr_run_thresholds); ++i) {
336 if (i == (ARRAY_SIZE(nr_run_thresholds) - 1))
337 nr_run_thresholds[i] = UINT_MAX;
338 else if (i < ARRAY_SIZE(default_thresholds))
339 nr_run_thresholds[i] = default_thresholds[i];
341 nr_run_thresholds[i] = i + 1 +
342 NR_FSHIFT / default_threshold_level;
345 mutex_lock(&runnables_lock);
346 runnables_state = RUNNING;
347 mutex_unlock(&runnables_lock);
349 runnables_avg_sampler(0);
354 struct cpuquiet_governor runnables_governor = {
356 .start = runnables_start,
357 .device_free_notification = runnables_device_free,
358 .device_busy_notification = runnables_device_busy,
359 .stop = runnables_stop,
360 .owner = THIS_MODULE,
363 static int __init init_runnables(void)
365 return cpuquiet_register_governor(&runnables_governor);
368 static void __exit exit_runnables(void)
370 cpuquiet_unregister_governor(&runnables_governor);
373 MODULE_LICENSE("GPL");
374 #ifdef CONFIG_CPUQUIET_DEFAULT_GOV_RUNNABLE
375 fs_initcall(init_runnables);
377 module_init(init_runnables);
379 module_exit(exit_runnables);