cpufreq: interactive: base hispeed bump on target freq, not actual
[firefly-linux-kernel-4.4.55.git] / drivers / cpufreq / cpufreq_interactive.c
1 /*
2  * drivers/cpufreq/cpufreq_interactive.c
3  *
4  * Copyright (C) 2010 Google, Inc.
5  *
6  * This software is licensed under the terms of the GNU General Public
7  * License version 2, as published by the Free Software Foundation, and
8  * may be copied, distributed, and modified under those terms.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * Author: Mike Chan (mike@android.com)
16  *
17  */
18
19 #include <linux/cpu.h>
20 #include <linux/cpumask.h>
21 #include <linux/cpufreq.h>
22 #include <linux/module.h>
23 #include <linux/mutex.h>
24 #include <linux/sched.h>
25 #include <linux/sched/rt.h>
26 #include <linux/tick.h>
27 #include <linux/time.h>
28 #include <linux/timer.h>
29 #include <linux/workqueue.h>
30 #include <linux/kthread.h>
31 #include <linux/mutex.h>
32
33 #define CREATE_TRACE_POINTS
34 #include <trace/events/cpufreq_interactive.h>
35
36 #include <asm/cputime.h>
37
38 static atomic_t active_count = ATOMIC_INIT(0);
39
40 struct cpufreq_interactive_cpuinfo {
41         struct timer_list cpu_timer;
42         int timer_idlecancel;
43         u64 time_in_idle;
44         u64 idle_exit_time;
45         u64 timer_run_time;
46         int idling;
47         u64 target_set_time;
48         u64 target_set_time_in_idle;
49         struct cpufreq_policy *policy;
50         struct cpufreq_frequency_table *freq_table;
51         unsigned int target_freq;
52         int governor_enabled;
53 };
54
55 static DEFINE_PER_CPU(struct cpufreq_interactive_cpuinfo, cpuinfo);
56
57 /* Workqueues handle frequency scaling */
58 static struct task_struct *up_task;
59 static struct workqueue_struct *down_wq;
60 static struct work_struct freq_scale_down_work;
61 static cpumask_t up_cpumask;
62 static spinlock_t up_cpumask_lock;
63 static cpumask_t down_cpumask;
64 static spinlock_t down_cpumask_lock;
65 static struct mutex set_speed_lock;
66
67 /* Hi speed to bump to from lo speed when load burst (default max) */
68 static u64 hispeed_freq;
69
70 /* Go to hi speed when CPU load at or above this value. */
71 #define DEFAULT_GO_HISPEED_LOAD 85
72 static unsigned long go_hispeed_load;
73
74 /*
75  * The minimum amount of time to spend at a frequency before we can ramp down.
76  */
77 #define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
78 static unsigned long min_sample_time;
79
80 /*
81  * The sample rate of the timer used to increase frequency
82  */
83 #define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC)
84 static unsigned long timer_rate;
85
86 /*
87  * Wait this long before raising speed above hispeed, by default a single
88  * timer interval.
89  */
90 #define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE
91 static unsigned long above_hispeed_delay_val;
92
93 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
94                 unsigned int event);
95
96 #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
97 static
98 #endif
99 struct cpufreq_governor cpufreq_gov_interactive = {
100         .name = "interactive",
101         .governor = cpufreq_governor_interactive,
102         .max_transition_latency = 10000000,
103         .owner = THIS_MODULE,
104 };
105
106 static void cpufreq_interactive_timer(unsigned long data)
107 {
108         unsigned int delta_idle;
109         unsigned int delta_time;
110         int cpu_load;
111         int load_since_change;
112         u64 time_in_idle;
113         u64 idle_exit_time;
114         struct cpufreq_interactive_cpuinfo *pcpu =
115                 &per_cpu(cpuinfo, data);
116         u64 now_idle;
117         unsigned int new_freq;
118         unsigned int index;
119         unsigned long flags;
120
121         smp_rmb();
122
123         if (!pcpu->governor_enabled)
124                 goto exit;
125
126         /*
127          * Once pcpu->timer_run_time is updated to >= pcpu->idle_exit_time,
128          * this lets idle exit know the current idle time sample has
129          * been processed, and idle exit can generate a new sample and
130          * re-arm the timer.  This prevents a concurrent idle
131          * exit on that CPU from writing a new set of info at the same time
132          * the timer function runs (the timer function can't use that info
133          * until more time passes).
134          */
135         time_in_idle = pcpu->time_in_idle;
136         idle_exit_time = pcpu->idle_exit_time;
137         now_idle = get_cpu_idle_time_us(data, &pcpu->timer_run_time);
138         smp_wmb();
139
140         /* If we raced with cancelling a timer, skip. */
141         if (!idle_exit_time)
142                 goto exit;
143
144         delta_idle = (unsigned int)(now_idle - time_in_idle);
145         delta_time = (unsigned int)(pcpu->timer_run_time - idle_exit_time);
146
147         /*
148          * If timer ran less than 1ms after short-term sample started, retry.
149          */
150         if (delta_time < 1000)
151                 goto rearm;
152
153         if (delta_idle > delta_time)
154                 cpu_load = 0;
155         else
156                 cpu_load = 100 * (delta_time - delta_idle) / delta_time;
157
158         delta_idle = (unsigned int)(now_idle - pcpu->target_set_time_in_idle);
159         delta_time = (unsigned int)(pcpu->timer_run_time -
160                                     pcpu->target_set_time);
161
162         if ((delta_time == 0) || (delta_idle > delta_time))
163                 load_since_change = 0;
164         else
165                 load_since_change =
166                         100 * (delta_time - delta_idle) / delta_time;
167
168         /*
169          * Choose greater of short-term load (since last idle timer
170          * started or timer function re-armed itself) or long-term load
171          * (since last frequency change).
172          */
173         if (load_since_change > cpu_load)
174                 cpu_load = load_since_change;
175
176         if (cpu_load >= go_hispeed_load) {
177                 if (pcpu->target_freq <= pcpu->policy->min) {
178                         new_freq = hispeed_freq;
179                 } else {
180                         new_freq = pcpu->policy->max * cpu_load / 100;
181
182                         if (new_freq < hispeed_freq)
183                                 new_freq = hispeed_freq;
184
185                         if (pcpu->target_freq == hispeed_freq &&
186                             new_freq > hispeed_freq &&
187                             pcpu->timer_run_time - pcpu->target_set_time
188                             < above_hispeed_delay_val) {
189                                 trace_cpufreq_interactive_notyet(data, cpu_load,
190                                                                  pcpu->target_freq,
191                                                                  new_freq);
192                                 goto rearm;
193                         }
194                 }
195         } else {
196                 new_freq = pcpu->policy->max * cpu_load / 100;
197         }
198
199         if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
200                                            new_freq, CPUFREQ_RELATION_H,
201                                            &index)) {
202                 pr_warn_once("timer %d: cpufreq_frequency_table_target error\n",
203                              (int) data);
204                 goto rearm;
205         }
206
207         new_freq = pcpu->freq_table[index].frequency;
208
209         /*
210          * Do not scale down unless we have been at this frequency for the
211          * minimum sample time.
212          */
213         if (new_freq < pcpu->target_freq) {
214                 if (pcpu->timer_run_time - pcpu->target_set_time
215                     < min_sample_time) {
216                         trace_cpufreq_interactive_notyet(data, cpu_load,
217                                          pcpu->target_freq, new_freq);
218                         goto rearm;
219                 }
220         }
221
222         pcpu->target_set_time_in_idle = now_idle;
223         pcpu->target_set_time = pcpu->timer_run_time;
224
225         if (pcpu->target_freq == new_freq) {
226                 trace_cpufreq_interactive_already(data, cpu_load,
227                                                   pcpu->target_freq, new_freq);
228                 goto rearm_if_notmax;
229         }
230
231         trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
232                                          new_freq);
233
234         if (new_freq < pcpu->target_freq) {
235                 pcpu->target_freq = new_freq;
236                 spin_lock_irqsave(&down_cpumask_lock, flags);
237                 cpumask_set_cpu(data, &down_cpumask);
238                 spin_unlock_irqrestore(&down_cpumask_lock, flags);
239                 queue_work(down_wq, &freq_scale_down_work);
240         } else {
241                 pcpu->target_freq = new_freq;
242                 spin_lock_irqsave(&up_cpumask_lock, flags);
243                 cpumask_set_cpu(data, &up_cpumask);
244                 spin_unlock_irqrestore(&up_cpumask_lock, flags);
245                 wake_up_process(up_task);
246         }
247
248 rearm_if_notmax:
249         /*
250          * Already set max speed and don't see a need to change that,
251          * wait until next idle to re-evaluate, don't need timer.
252          */
253         if (pcpu->target_freq == pcpu->policy->max)
254                 goto exit;
255
256 rearm:
257         if (!timer_pending(&pcpu->cpu_timer)) {
258                 /*
259                  * If already at min: if that CPU is idle, don't set timer.
260                  * Else cancel the timer if that CPU goes idle.  We don't
261                  * need to re-evaluate speed until the next idle exit.
262                  */
263                 if (pcpu->target_freq == pcpu->policy->min) {
264                         smp_rmb();
265
266                         if (pcpu->idling)
267                                 goto exit;
268
269                         pcpu->timer_idlecancel = 1;
270                 }
271
272                 pcpu->time_in_idle = get_cpu_idle_time_us(
273                         data, &pcpu->idle_exit_time);
274                 mod_timer(&pcpu->cpu_timer,
275                           jiffies + usecs_to_jiffies(timer_rate));
276         }
277
278 exit:
279         return;
280 }
281
282 static void cpufreq_interactive_idle_start(void)
283 {
284         struct cpufreq_interactive_cpuinfo *pcpu =
285                 &per_cpu(cpuinfo, smp_processor_id());
286         int pending;
287
288         if (!pcpu->governor_enabled)
289                 return;
290
291         pcpu->idling = 1;
292         smp_wmb();
293         pending = timer_pending(&pcpu->cpu_timer);
294
295         if (pcpu->target_freq != pcpu->policy->min) {
296 #ifdef CONFIG_SMP
297                 /*
298                  * Entering idle while not at lowest speed.  On some
299                  * platforms this can hold the other CPU(s) at that speed
300                  * even though the CPU is idle. Set a timer to re-evaluate
301                  * speed so this idle CPU doesn't hold the other CPUs above
302                  * min indefinitely.  This should probably be a quirk of
303                  * the CPUFreq driver.
304                  */
305                 if (!pending) {
306                         pcpu->time_in_idle = get_cpu_idle_time_us(
307                                 smp_processor_id(), &pcpu->idle_exit_time);
308                         pcpu->timer_idlecancel = 0;
309                         mod_timer(&pcpu->cpu_timer,
310                                   jiffies + usecs_to_jiffies(timer_rate));
311                 }
312 #endif
313         } else {
314                 /*
315                  * If at min speed and entering idle after load has
316                  * already been evaluated, and a timer has been set just in
317                  * case the CPU suddenly goes busy, cancel that timer.  The
318                  * CPU didn't go busy; we'll recheck things upon idle exit.
319                  */
320                 if (pending && pcpu->timer_idlecancel) {
321                         del_timer(&pcpu->cpu_timer);
322                         /*
323                          * Ensure last timer run time is after current idle
324                          * sample start time, so next idle exit will always
325                          * start a new idle sampling period.
326                          */
327                         pcpu->idle_exit_time = 0;
328                         pcpu->timer_idlecancel = 0;
329                 }
330         }
331
332 }
333
334 static void cpufreq_interactive_idle_end(void)
335 {
336         struct cpufreq_interactive_cpuinfo *pcpu =
337                 &per_cpu(cpuinfo, smp_processor_id());
338
339         pcpu->idling = 0;
340         smp_wmb();
341
342         /*
343          * Arm the timer for 1-2 ticks later if not already, and if the timer
344          * function has already processed the previous load sampling
345          * interval.  (If the timer is not pending but has not processed
346          * the previous interval, it is probably racing with us on another
347          * CPU.  Let it compute load based on the previous sample and then
348          * re-arm the timer for another interval when it's done, rather
349          * than updating the interval start time to be "now", which doesn't
350          * give the timer function enough time to make a decision on this
351          * run.)
352          */
353         if (timer_pending(&pcpu->cpu_timer) == 0 &&
354             pcpu->timer_run_time >= pcpu->idle_exit_time &&
355             pcpu->governor_enabled) {
356                 pcpu->time_in_idle =
357                         get_cpu_idle_time_us(smp_processor_id(),
358                                              &pcpu->idle_exit_time);
359                 pcpu->timer_idlecancel = 0;
360                 mod_timer(&pcpu->cpu_timer,
361                           jiffies + usecs_to_jiffies(timer_rate));
362         }
363
364 }
365
366 static int cpufreq_interactive_up_task(void *data)
367 {
368         unsigned int cpu;
369         cpumask_t tmp_mask;
370         unsigned long flags;
371         struct cpufreq_interactive_cpuinfo *pcpu;
372
373         while (1) {
374                 set_current_state(TASK_INTERRUPTIBLE);
375                 spin_lock_irqsave(&up_cpumask_lock, flags);
376
377                 if (cpumask_empty(&up_cpumask)) {
378                         spin_unlock_irqrestore(&up_cpumask_lock, flags);
379                         schedule();
380
381                         if (kthread_should_stop())
382                                 break;
383
384                         spin_lock_irqsave(&up_cpumask_lock, flags);
385                 }
386
387                 set_current_state(TASK_RUNNING);
388                 tmp_mask = up_cpumask;
389                 cpumask_clear(&up_cpumask);
390                 spin_unlock_irqrestore(&up_cpumask_lock, flags);
391
392                 for_each_cpu(cpu, &tmp_mask) {
393                         unsigned int j;
394                         unsigned int max_freq = 0;
395
396                         pcpu = &per_cpu(cpuinfo, cpu);
397                         smp_rmb();
398
399                         if (!pcpu->governor_enabled)
400                                 continue;
401
402                         mutex_lock(&set_speed_lock);
403
404                         for_each_cpu(j, pcpu->policy->cpus) {
405                                 struct cpufreq_interactive_cpuinfo *pjcpu =
406                                         &per_cpu(cpuinfo, j);
407
408                                 if (pjcpu->target_freq > max_freq)
409                                         max_freq = pjcpu->target_freq;
410                         }
411
412                         if (max_freq != pcpu->policy->cur)
413                                 __cpufreq_driver_target(pcpu->policy,
414                                                         max_freq,
415                                                         CPUFREQ_RELATION_H);
416                         mutex_unlock(&set_speed_lock);
417                         trace_cpufreq_interactive_up(cpu, pcpu->target_freq,
418                                                      pcpu->policy->cur);
419                 }
420         }
421
422         return 0;
423 }
424
425 static void cpufreq_interactive_freq_down(struct work_struct *work)
426 {
427         unsigned int cpu;
428         cpumask_t tmp_mask;
429         unsigned long flags;
430         struct cpufreq_interactive_cpuinfo *pcpu;
431
432         spin_lock_irqsave(&down_cpumask_lock, flags);
433         tmp_mask = down_cpumask;
434         cpumask_clear(&down_cpumask);
435         spin_unlock_irqrestore(&down_cpumask_lock, flags);
436
437         for_each_cpu(cpu, &tmp_mask) {
438                 unsigned int j;
439                 unsigned int max_freq = 0;
440
441                 pcpu = &per_cpu(cpuinfo, cpu);
442                 smp_rmb();
443
444                 if (!pcpu->governor_enabled)
445                         continue;
446
447                 mutex_lock(&set_speed_lock);
448
449                 for_each_cpu(j, pcpu->policy->cpus) {
450                         struct cpufreq_interactive_cpuinfo *pjcpu =
451                                 &per_cpu(cpuinfo, j);
452
453                         if (pjcpu->target_freq > max_freq)
454                                 max_freq = pjcpu->target_freq;
455                 }
456
457                 if (max_freq != pcpu->policy->cur)
458                         __cpufreq_driver_target(pcpu->policy, max_freq,
459                                                 CPUFREQ_RELATION_H);
460
461                 mutex_unlock(&set_speed_lock);
462                 trace_cpufreq_interactive_down(cpu, pcpu->target_freq,
463                                                pcpu->policy->cur);
464         }
465 }
466
467 static ssize_t show_hispeed_freq(struct kobject *kobj,
468                                  struct attribute *attr, char *buf)
469 {
470         return sprintf(buf, "%llu\n", hispeed_freq);
471 }
472
473 static ssize_t store_hispeed_freq(struct kobject *kobj,
474                                   struct attribute *attr, const char *buf,
475                                   size_t count)
476 {
477         int ret;
478         u64 val;
479
480         ret = strict_strtoull(buf, 0, &val);
481         if (ret < 0)
482                 return ret;
483         hispeed_freq = val;
484         return count;
485 }
486
487 static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644,
488                 show_hispeed_freq, store_hispeed_freq);
489
490
491 static ssize_t show_go_hispeed_load(struct kobject *kobj,
492                                      struct attribute *attr, char *buf)
493 {
494         return sprintf(buf, "%lu\n", go_hispeed_load);
495 }
496
497 static ssize_t store_go_hispeed_load(struct kobject *kobj,
498                         struct attribute *attr, const char *buf, size_t count)
499 {
500         int ret;
501         unsigned long val;
502
503         ret = strict_strtoul(buf, 0, &val);
504         if (ret < 0)
505                 return ret;
506         go_hispeed_load = val;
507         return count;
508 }
509
510 static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644,
511                 show_go_hispeed_load, store_go_hispeed_load);
512
513 static ssize_t show_min_sample_time(struct kobject *kobj,
514                                 struct attribute *attr, char *buf)
515 {
516         return sprintf(buf, "%lu\n", min_sample_time);
517 }
518
519 static ssize_t store_min_sample_time(struct kobject *kobj,
520                         struct attribute *attr, const char *buf, size_t count)
521 {
522         int ret;
523         unsigned long val;
524
525         ret = strict_strtoul(buf, 0, &val);
526         if (ret < 0)
527                 return ret;
528         min_sample_time = val;
529         return count;
530 }
531
532 static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
533                 show_min_sample_time, store_min_sample_time);
534
535 static ssize_t show_above_hispeed_delay(struct kobject *kobj,
536                                         struct attribute *attr, char *buf)
537 {
538         return sprintf(buf, "%lu\n", above_hispeed_delay_val);
539 }
540
541 static ssize_t store_above_hispeed_delay(struct kobject *kobj,
542                                          struct attribute *attr,
543                                          const char *buf, size_t count)
544 {
545         int ret;
546         unsigned long val;
547
548         ret = strict_strtoul(buf, 0, &val);
549         if (ret < 0)
550                 return ret;
551         above_hispeed_delay_val = val;
552         return count;
553 }
554
555 define_one_global_rw(above_hispeed_delay);
556
557 static ssize_t show_timer_rate(struct kobject *kobj,
558                         struct attribute *attr, char *buf)
559 {
560         return sprintf(buf, "%lu\n", timer_rate);
561 }
562
563 static ssize_t store_timer_rate(struct kobject *kobj,
564                         struct attribute *attr, const char *buf, size_t count)
565 {
566         int ret;
567         unsigned long val;
568
569         ret = strict_strtoul(buf, 0, &val);
570         if (ret < 0)
571                 return ret;
572         timer_rate = val;
573         return count;
574 }
575
576 static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
577                 show_timer_rate, store_timer_rate);
578
579 static struct attribute *interactive_attributes[] = {
580         &hispeed_freq_attr.attr,
581         &go_hispeed_load_attr.attr,
582         &above_hispeed_delay.attr,
583         &min_sample_time_attr.attr,
584         &timer_rate_attr.attr,
585         NULL,
586 };
587
588 static struct attribute_group interactive_attr_group = {
589         .attrs = interactive_attributes,
590         .name = "interactive",
591 };
592
593 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
594                 unsigned int event)
595 {
596         int rc;
597         unsigned int j;
598         struct cpufreq_interactive_cpuinfo *pcpu;
599         struct cpufreq_frequency_table *freq_table;
600
601         switch (event) {
602         case CPUFREQ_GOV_START:
603                 if (!cpu_online(policy->cpu))
604                         return -EINVAL;
605
606                 freq_table =
607                         cpufreq_frequency_get_table(policy->cpu);
608
609                 for_each_cpu(j, policy->cpus) {
610                         pcpu = &per_cpu(cpuinfo, j);
611                         pcpu->policy = policy;
612                         pcpu->target_freq = policy->cur;
613                         pcpu->freq_table = freq_table;
614                         pcpu->target_set_time_in_idle =
615                                 get_cpu_idle_time_us(j,
616                                              &pcpu->target_set_time);
617                         pcpu->governor_enabled = 1;
618                         smp_wmb();
619                 }
620
621                 if (!hispeed_freq)
622                         hispeed_freq = policy->max;
623
624                 /*
625                  * Do not register the idle hook and create sysfs
626                  * entries if we have already done so.
627                  */
628                 if (atomic_inc_return(&active_count) > 1)
629                         return 0;
630
631                 rc = sysfs_create_group(cpufreq_global_kobject,
632                                 &interactive_attr_group);
633                 if (rc)
634                         return rc;
635
636                 break;
637
638         case CPUFREQ_GOV_STOP:
639                 for_each_cpu(j, policy->cpus) {
640                         pcpu = &per_cpu(cpuinfo, j);
641                         pcpu->governor_enabled = 0;
642                         smp_wmb();
643                         del_timer_sync(&pcpu->cpu_timer);
644
645                         /*
646                          * Reset idle exit time since we may cancel the timer
647                          * before it can run after the last idle exit time,
648                          * to avoid tripping the check in idle exit for a timer
649                          * that is trying to run.
650                          */
651                         pcpu->idle_exit_time = 0;
652                 }
653
654                 flush_work(&freq_scale_down_work);
655                 if (atomic_dec_return(&active_count) > 0)
656                         return 0;
657
658                 sysfs_remove_group(cpufreq_global_kobject,
659                                 &interactive_attr_group);
660
661                 break;
662
663         case CPUFREQ_GOV_LIMITS:
664                 if (policy->max < policy->cur)
665                         __cpufreq_driver_target(policy,
666                                         policy->max, CPUFREQ_RELATION_H);
667                 else if (policy->min > policy->cur)
668                         __cpufreq_driver_target(policy,
669                                         policy->min, CPUFREQ_RELATION_L);
670                 break;
671         }
672         return 0;
673 }
674
675 static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
676                                              unsigned long val,
677                                              void *data)
678 {
679         switch (val) {
680         case IDLE_START:
681                 cpufreq_interactive_idle_start();
682                 break;
683         case IDLE_END:
684                 cpufreq_interactive_idle_end();
685                 break;
686         }
687
688         return 0;
689 }
690
691 static struct notifier_block cpufreq_interactive_idle_nb = {
692         .notifier_call = cpufreq_interactive_idle_notifier,
693 };
694
695 static int __init cpufreq_interactive_init(void)
696 {
697         unsigned int i;
698         struct cpufreq_interactive_cpuinfo *pcpu;
699         struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
700
701         go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
702         min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
703         above_hispeed_delay_val = DEFAULT_ABOVE_HISPEED_DELAY;
704         timer_rate = DEFAULT_TIMER_RATE;
705
706         /* Initalize per-cpu timers */
707         for_each_possible_cpu(i) {
708                 pcpu = &per_cpu(cpuinfo, i);
709                 init_timer(&pcpu->cpu_timer);
710                 pcpu->cpu_timer.function = cpufreq_interactive_timer;
711                 pcpu->cpu_timer.data = i;
712         }
713
714         up_task = kthread_create(cpufreq_interactive_up_task, NULL,
715                                  "kinteractiveup");
716         if (IS_ERR(up_task))
717                 return PTR_ERR(up_task);
718
719         sched_setscheduler_nocheck(up_task, SCHED_FIFO, &param);
720         get_task_struct(up_task);
721
722         /* No rescuer thread, bind to CPU queuing the work for possibly
723            warm cache (probably doesn't matter much). */
724         down_wq = alloc_workqueue("knteractive_down", 0, 1);
725
726         if (!down_wq)
727                 goto err_freeuptask;
728
729         INIT_WORK(&freq_scale_down_work,
730                   cpufreq_interactive_freq_down);
731
732         spin_lock_init(&up_cpumask_lock);
733         spin_lock_init(&down_cpumask_lock);
734         mutex_init(&set_speed_lock);
735
736         idle_notifier_register(&cpufreq_interactive_idle_nb);
737
738         return cpufreq_register_governor(&cpufreq_gov_interactive);
739
740 err_freeuptask:
741         put_task_struct(up_task);
742         return -ENOMEM;
743 }
744
745 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
746 fs_initcall(cpufreq_interactive_init);
747 #else
748 module_init(cpufreq_interactive_init);
749 #endif
750
751 static void __exit cpufreq_interactive_exit(void)
752 {
753         cpufreq_unregister_governor(&cpufreq_gov_interactive);
754         kthread_stop(up_task);
755         put_task_struct(up_task);
756         destroy_workqueue(down_wq);
757 }
758
759 module_exit(cpufreq_interactive_exit);
760
761 MODULE_AUTHOR("Mike Chan <mike@android.com>");
762 MODULE_DESCRIPTION("'cpufreq_interactive' - A cpufreq governor for "
763         "Latency sensitive workloads");
764 MODULE_LICENSE("GPL");