2 * Copyright (C) 2015 Michael Turquette <mturquette@linaro.org>
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
9 #include <linux/cpufreq.h>
10 #include <linux/module.h>
11 #include <linux/kthread.h>
12 #include <linux/percpu.h>
13 #include <linux/irq_work.h>
14 #include <linux/delay.h>
15 #include <linux/string.h>
17 #define CREATE_TRACE_POINTS
18 #include <trace/events/cpufreq_sched.h>
22 #define THROTTLE_DOWN_NSEC 50000000 /* 50ms default */
23 #define THROTTLE_UP_NSEC 500000 /* 500us default */
25 struct static_key __read_mostly __sched_freq = STATIC_KEY_INIT_FALSE;
26 static bool __read_mostly cpufreq_driver_slow;
28 #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_SCHED
29 static struct cpufreq_governor cpufreq_gov_sched;
32 static DEFINE_PER_CPU(unsigned long, enabled);
33 DEFINE_PER_CPU(struct sched_capacity_reqs, cpu_sched_capacity_reqs);
36 * gov_data - per-policy data internal to the governor
37 * @up_throttle: next throttling period expiry if increasing OPP
38 * @down_throttle: next throttling period expiry if decreasing OPP
39 * @up_throttle_nsec: throttle period length in nanoseconds if increasing OPP
40 * @down_throttle_nsec: throttle period length in nanoseconds if decreasing OPP
41 * @task: worker thread for dvfs transition that may block/sleep
42 * @irq_work: callback used to wake up worker thread
43 * @requested_freq: last frequency requested by the sched governor
45 * struct gov_data is the per-policy cpufreq_sched-specific data structure. A
46 * per-policy instance of it is created when the cpufreq_sched governor receives
47 * the CPUFREQ_GOV_START condition and a pointer to it exists in the gov_data
48 * member of struct cpufreq_policy.
50 * Readers of this data must call down_read(policy->rwsem). Writers must
51 * call down_write(policy->rwsem).
55 ktime_t down_throttle;
56 unsigned int up_throttle_nsec;
57 unsigned int down_throttle_nsec;
58 struct task_struct *task;
59 struct irq_work irq_work;
60 unsigned int requested_freq;
63 static void cpufreq_sched_try_driver_target(struct cpufreq_policy *policy,
66 struct gov_data *gd = policy->governor_data;
68 /* avoid race with cpufreq_sched_stop */
69 if (!down_write_trylock(&policy->rwsem))
72 __cpufreq_driver_target(policy, freq, CPUFREQ_RELATION_L);
74 gd->up_throttle = ktime_add_ns(ktime_get(), gd->up_throttle_nsec);
75 gd->down_throttle = ktime_add_ns(ktime_get(), gd->down_throttle_nsec);
76 up_write(&policy->rwsem);
79 static bool finish_last_request(struct gov_data *gd, unsigned int cur_freq)
81 ktime_t now = ktime_get();
83 ktime_t throttle = gd->requested_freq < cur_freq ?
84 gd->down_throttle : gd->up_throttle;
86 if (ktime_after(now, throttle))
90 int usec_left = ktime_to_ns(ktime_sub(throttle, now));
92 usec_left /= NSEC_PER_USEC;
93 trace_cpufreq_sched_throttled(usec_left);
94 usleep_range(usec_left, usec_left + 100);
96 if (ktime_after(now, throttle))
102 * we pass in struct cpufreq_policy. This is safe because changing out the
103 * policy requires a call to __cpufreq_governor(policy, CPUFREQ_GOV_STOP),
104 * which tears down all of the data structures and __cpufreq_governor(policy,
105 * CPUFREQ_GOV_START) will do a full rebuild, including this kthread with the
108 static int cpufreq_sched_thread(void *data)
110 struct sched_param param;
111 struct cpufreq_policy *policy;
113 unsigned int new_request = 0;
114 unsigned int last_request = 0;
117 policy = (struct cpufreq_policy *) data;
118 gd = policy->governor_data;
120 param.sched_priority = 50;
121 ret = sched_setscheduler_nocheck(gd->task, SCHED_FIFO, ¶m);
123 pr_warn("%s: failed to set SCHED_FIFO\n", __func__);
126 pr_debug("%s: kthread (%d) set to SCHED_FIFO\n",
127 __func__, gd->task->pid);
131 new_request = gd->requested_freq;
132 if (new_request == last_request) {
133 set_current_state(TASK_INTERRUPTIBLE);
137 * if the frequency thread sleeps while waiting to be
138 * unthrottled, start over to check for a newer request
140 if (finish_last_request(gd, policy->cur))
142 last_request = new_request;
143 cpufreq_sched_try_driver_target(policy, new_request);
145 } while (!kthread_should_stop());
150 static void cpufreq_sched_irq_work(struct irq_work *irq_work)
154 gd = container_of(irq_work, struct gov_data, irq_work);
158 wake_up_process(gd->task);
161 static void update_fdomain_capacity_request(int cpu)
163 unsigned int freq_new, index_new, cpu_tmp;
164 struct cpufreq_policy *policy;
166 unsigned long capacity = 0;
169 * Avoid grabbing the policy if possible. A test is still
170 * required after locking the CPU's policy to avoid racing
171 * with the governor changing.
173 if (!per_cpu(enabled, cpu))
176 policy = cpufreq_cpu_get(cpu);
177 if (IS_ERR_OR_NULL(policy))
180 if (policy->governor != &cpufreq_gov_sched ||
181 !policy->governor_data)
184 gd = policy->governor_data;
186 /* find max capacity requested by cpus in this policy */
187 for_each_cpu(cpu_tmp, policy->cpus) {
188 struct sched_capacity_reqs *scr;
190 scr = &per_cpu(cpu_sched_capacity_reqs, cpu_tmp);
191 capacity = max(capacity, scr->total);
194 /* Convert the new maximum capacity request into a cpu frequency */
195 freq_new = capacity * policy->max >> SCHED_CAPACITY_SHIFT;
196 if (cpufreq_frequency_table_target(policy, policy->freq_table,
197 freq_new, CPUFREQ_RELATION_L,
200 freq_new = policy->freq_table[index_new].frequency;
202 if (freq_new > policy->max)
203 freq_new = policy->max;
205 if (freq_new < policy->min)
206 freq_new = policy->min;
208 trace_cpufreq_sched_request_opp(cpu, capacity, freq_new,
210 if (freq_new == gd->requested_freq)
213 gd->requested_freq = freq_new;
216 * Throttling is not yet supported on platforms with fast cpufreq
219 if (cpufreq_driver_slow)
220 irq_work_queue_on(&gd->irq_work, cpu);
222 cpufreq_sched_try_driver_target(policy, freq_new);
225 cpufreq_cpu_put(policy);
228 void update_cpu_capacity_request(int cpu, bool request)
230 unsigned long new_capacity;
231 struct sched_capacity_reqs *scr;
233 /* The rq lock serializes access to the CPU's sched_capacity_reqs. */
234 lockdep_assert_held(&cpu_rq(cpu)->lock);
236 scr = &per_cpu(cpu_sched_capacity_reqs, cpu);
238 new_capacity = scr->cfs + scr->rt;
239 new_capacity = new_capacity * capacity_margin
240 / SCHED_CAPACITY_SCALE;
241 new_capacity += scr->dl;
243 if (new_capacity == scr->total)
246 trace_cpufreq_sched_update_capacity(cpu, request, scr, new_capacity);
248 scr->total = new_capacity;
250 update_fdomain_capacity_request(cpu);
253 static inline void set_sched_freq(void)
255 static_key_slow_inc(&__sched_freq);
258 static inline void clear_sched_freq(void)
260 static_key_slow_dec(&__sched_freq);
263 static struct attribute_group sched_attr_group_gov_pol;
264 static struct attribute_group *get_sysfs_attr(void)
266 return &sched_attr_group_gov_pol;
269 static int cpufreq_sched_policy_init(struct cpufreq_policy *policy)
275 for_each_cpu(cpu, policy->cpus)
276 memset(&per_cpu(cpu_sched_capacity_reqs, cpu), 0,
277 sizeof(struct sched_capacity_reqs));
279 gd = kzalloc(sizeof(*gd), GFP_KERNEL);
283 gd->up_throttle_nsec = policy->cpuinfo.transition_latency ?
284 policy->cpuinfo.transition_latency :
286 gd->down_throttle_nsec = THROTTLE_DOWN_NSEC;
287 pr_debug("%s: throttle threshold = %u [ns]\n",
288 __func__, gd->up_throttle_nsec);
290 rc = sysfs_create_group(get_governor_parent_kobj(policy), get_sysfs_attr());
292 pr_err("%s: couldn't create sysfs attributes: %d\n", __func__, rc);
296 if (cpufreq_driver_is_slow()) {
297 cpufreq_driver_slow = true;
298 gd->task = kthread_create(cpufreq_sched_thread, policy,
300 cpumask_first(policy->related_cpus));
301 if (IS_ERR_OR_NULL(gd->task)) {
302 pr_err("%s: failed to create kschedfreq thread\n",
306 get_task_struct(gd->task);
307 kthread_bind_mask(gd->task, policy->related_cpus);
308 wake_up_process(gd->task);
309 init_irq_work(&gd->irq_work, cpufreq_sched_irq_work);
312 policy->governor_data = gd;
322 static int cpufreq_sched_policy_exit(struct cpufreq_policy *policy)
324 struct gov_data *gd = policy->governor_data;
327 if (cpufreq_driver_slow) {
328 kthread_stop(gd->task);
329 put_task_struct(gd->task);
332 sysfs_remove_group(get_governor_parent_kobj(policy), get_sysfs_attr());
334 policy->governor_data = NULL;
340 static int cpufreq_sched_start(struct cpufreq_policy *policy)
344 for_each_cpu(cpu, policy->cpus)
345 per_cpu(enabled, cpu) = 1;
350 static void cpufreq_sched_limits(struct cpufreq_policy *policy)
352 unsigned int clamp_freq;
353 struct gov_data *gd = policy->governor_data;;
355 pr_debug("limit event for cpu %u: %u - %u kHz, currently %u kHz\n",
356 policy->cpu, policy->min, policy->max,
359 clamp_freq = clamp(gd->requested_freq, policy->min, policy->max);
361 if (policy->cur != clamp_freq)
362 __cpufreq_driver_target(policy, clamp_freq, CPUFREQ_RELATION_L);
365 static int cpufreq_sched_stop(struct cpufreq_policy *policy)
369 for_each_cpu(cpu, policy->cpus)
370 per_cpu(enabled, cpu) = 0;
375 static int cpufreq_sched_setup(struct cpufreq_policy *policy,
379 case CPUFREQ_GOV_POLICY_INIT:
380 return cpufreq_sched_policy_init(policy);
381 case CPUFREQ_GOV_POLICY_EXIT:
382 return cpufreq_sched_policy_exit(policy);
383 case CPUFREQ_GOV_START:
384 return cpufreq_sched_start(policy);
385 case CPUFREQ_GOV_STOP:
386 return cpufreq_sched_stop(policy);
387 case CPUFREQ_GOV_LIMITS:
388 cpufreq_sched_limits(policy);
395 static ssize_t show_up_throttle_nsec(struct gov_data *gd, char *buf)
397 return sprintf(buf, "%u\n", gd->up_throttle_nsec);
400 static ssize_t store_up_throttle_nsec(struct gov_data *gd,
401 const char *buf, size_t count)
404 long unsigned int val;
406 ret = kstrtoul(buf, 0, &val);
409 gd->up_throttle_nsec = val;
413 static ssize_t show_down_throttle_nsec(struct gov_data *gd, char *buf)
415 return sprintf(buf, "%u\n", gd->down_throttle_nsec);
418 static ssize_t store_down_throttle_nsec(struct gov_data *gd,
419 const char *buf, size_t count)
422 long unsigned int val;
424 ret = kstrtoul(buf, 0, &val);
427 gd->down_throttle_nsec = val;
432 * Create show/store routines
433 * - sys: One governor instance for complete SYSTEM
434 * - pol: One governor instance per struct cpufreq_policy
436 #define show_gov_pol_sys(file_name) \
437 static ssize_t show_##file_name##_gov_pol \
438 (struct cpufreq_policy *policy, char *buf) \
440 return show_##file_name(policy->governor_data, buf); \
443 #define store_gov_pol_sys(file_name) \
444 static ssize_t store_##file_name##_gov_pol \
445 (struct cpufreq_policy *policy, const char *buf, size_t count) \
447 return store_##file_name(policy->governor_data, buf, count); \
450 #define gov_pol_attr_rw(_name) \
451 static struct freq_attr _name##_gov_pol = \
452 __ATTR(_name, 0644, show_##_name##_gov_pol, store_##_name##_gov_pol)
454 #define show_store_gov_pol_sys(file_name) \
455 show_gov_pol_sys(file_name); \
456 store_gov_pol_sys(file_name)
457 #define tunable_handlers(file_name) \
458 show_gov_pol_sys(file_name); \
459 store_gov_pol_sys(file_name); \
460 gov_pol_attr_rw(file_name)
462 tunable_handlers(down_throttle_nsec);
463 tunable_handlers(up_throttle_nsec);
465 /* Per policy governor instance */
466 static struct attribute *sched_attributes_gov_pol[] = {
467 &up_throttle_nsec_gov_pol.attr,
468 &down_throttle_nsec_gov_pol.attr,
472 static struct attribute_group sched_attr_group_gov_pol = {
473 .attrs = sched_attributes_gov_pol,
477 #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_SCHED
480 struct cpufreq_governor cpufreq_gov_sched = {
482 .governor = cpufreq_sched_setup,
483 .owner = THIS_MODULE,
486 static int __init cpufreq_sched_init(void)
490 for_each_cpu(cpu, cpu_possible_mask)
491 per_cpu(enabled, cpu) = 0;
492 return cpufreq_register_governor(&cpufreq_gov_sched);
495 /* Try to make this the default governor */
496 fs_initcall(cpufreq_sched_init);