drivers/cpufreq/cpufreq_interactive.c

   1 /*
   2  * drivers/cpufreq/cpufreq_interactive.c
   3  *
   4  * Copyright (C) 2010 Google, Inc.
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * Author: Mike Chan (mike@android.com)
  16  *
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/cpumask.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/module.h>
  23 #include <linux/mutex.h>
  24 #include <linux/sched.h>
  25 #include <linux/sched/rt.h>
  26 #include <linux/tick.h>
  27 #include <linux/time.h>
  28 #include <linux/timer.h>
  29 #include <linux/workqueue.h>
  30 #include <linux/kthread.h>
  31 #include <linux/mutex.h>
  32 #include <linux/slab.h>
  33 #include <asm/cputime.h>
  34
  35 #define CREATE_TRACE_POINTS
  36 #include <trace/events/cpufreq_interactive.h>
  37
  38 static atomic_t active_count = ATOMIC_INIT(0);
  39
  40 struct cpufreq_interactive_cpuinfo {
  41         struct timer_list cpu_timer;
  42         int timer_idlecancel;
  43         u64 time_in_idle;
  44         u64 idle_exit_time;
  45         u64 target_set_time;
  46         u64 target_set_time_in_idle;
  47         struct cpufreq_policy *policy;
  48         struct cpufreq_frequency_table *freq_table;
  49         unsigned int target_freq;
  50         unsigned int floor_freq;
  51         u64 floor_validate_time;
  52         u64 hispeed_validate_time;
  53         int governor_enabled;
  54 };
  55
  56 static DEFINE_PER_CPU(struct cpufreq_interactive_cpuinfo, cpuinfo);
  57
  58 /* realtime thread handles frequency scaling */
  59 static struct task_struct *speedchange_task;
  60 static cpumask_t speedchange_cpumask;
  61 static spinlock_t speedchange_cpumask_lock;
  62
  63 /* Hi speed to bump to from lo speed when load burst (default max) */
  64 static unsigned int hispeed_freq;
  65
  66 /* Go to hi speed when CPU load at or above this value. */
  67 #define DEFAULT_GO_HISPEED_LOAD 85
  68 static unsigned long go_hispeed_load;
  69
  70 /*
  71  * The minimum amount of time to spend at a frequency before we can ramp down.
  72  */
  73 #define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
  74 static unsigned long min_sample_time;
  75
  76 /*
  77  * The sample rate of the timer used to increase frequency
  78  */
  79 #define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC)
  80 static unsigned long timer_rate;
  81
  82 /*
  83  * Wait this long before raising speed above hispeed, by default a single
  84  * timer interval.
  85  */
  86 #define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE
  87 static unsigned long above_hispeed_delay_val;
  88
  89 /*
  90  * Non-zero means longer-term speed boost active.
  91  */
  92
  93 static int boost_val;
  94
  95 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
  96                 unsigned int event);
  97
  98 #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
  99 static
 100 #endif
 101 struct cpufreq_governor cpufreq_gov_interactive = {
 102         .name = "interactive",
 103         .governor = cpufreq_governor_interactive,
 104         .max_transition_latency = 10000000,
 105         .owner = THIS_MODULE,
 106 };
 107
 108 static void cpufreq_interactive_timer(unsigned long data)
 109 {
 110         u64 now;
 111         unsigned int delta_idle;
 112         unsigned int delta_time;
 113         int cpu_load;
 114         int load_since_change;
 115         u64 time_in_idle;
 116         u64 idle_exit_time;
 117         struct cpufreq_interactive_cpuinfo *pcpu =
 118                 &per_cpu(cpuinfo, data);
 119         u64 now_idle;
 120         unsigned int new_freq;
 121         unsigned int index;
 122         unsigned long flags;
 123
 124         smp_rmb();
 125
 126         if (!pcpu->governor_enabled)
 127                 goto exit;
 128
 129         time_in_idle = pcpu->time_in_idle;
 130         idle_exit_time = pcpu->idle_exit_time;
 131         now_idle = get_cpu_idle_time_us(data, &now);
 132         delta_idle = (unsigned int)(now_idle - time_in_idle);
 133         delta_time = (unsigned int)(now - idle_exit_time);
 134
 135         /*
 136          * If timer ran less than 1ms after short-term sample started, retry.
 137          */
 138         if (delta_time < 1000)
 139                 goto rearm;
 140
 141         if (delta_idle > delta_time)
 142                 cpu_load = 0;
 143         else
 144                 cpu_load = 100 * (delta_time - delta_idle) / delta_time;
 145
 146         delta_idle = (unsigned int)(now_idle - pcpu->target_set_time_in_idle);
 147         delta_time = (unsigned int)(now - pcpu->target_set_time);
 148
 149         if ((delta_time == 0) || (delta_idle > delta_time))
 150                 load_since_change = 0;
 151         else
 152                 load_since_change =
 153                         100 * (delta_time - delta_idle) / delta_time;
 154
 155         /*
 156          * Choose greater of short-term load (since last idle timer
 157          * started or timer function re-armed itself) or long-term load
 158          * (since last frequency change).
 159          */
 160         if (load_since_change > cpu_load)
 161                 cpu_load = load_since_change;
 162
 163         if (cpu_load >= go_hispeed_load || boost_val) {
 164                 if (pcpu->target_freq < hispeed_freq &&
 165                     hispeed_freq < pcpu->policy->max) {
 166                         new_freq = hispeed_freq;
 167                 } else {
 168                         new_freq = pcpu->policy->max * cpu_load / 100;
 169
 170                         if (new_freq < hispeed_freq)
 171                                 new_freq = hispeed_freq;
 172
 173                         if (pcpu->target_freq == hispeed_freq &&
 174                             new_freq > hispeed_freq &&
 175                             now - pcpu->hispeed_validate_time
 176                             < above_hispeed_delay_val) {
 177                                 trace_cpufreq_interactive_notyet(data, cpu_load,
 178                                                                  pcpu->target_freq,
 179                                                                  new_freq);
 180                                 goto rearm;
 181                         }
 182                 }
 183         } else {
 184                 new_freq = hispeed_freq * cpu_load / 100;
 185         }
 186
 187         if (new_freq <= hispeed_freq)
 188                 pcpu->hispeed_validate_time = now;
 189
 190         if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
 191                                            new_freq, CPUFREQ_RELATION_H,
 192                                            &index)) {
 193                 pr_warn_once("timer %d: cpufreq_frequency_table_target error\n",
 194                              (int) data);
 195                 goto rearm;
 196         }
 197
 198         new_freq = pcpu->freq_table[index].frequency;
 199
 200         /*
 201          * Do not scale below floor_freq unless we have been at or above the
 202          * floor frequency for the minimum sample time since last validated.
 203          */
 204         if (new_freq < pcpu->floor_freq) {
 205                 if (now - pcpu->floor_validate_time < min_sample_time) {
 206                         trace_cpufreq_interactive_notyet(data, cpu_load,
 207                                          pcpu->target_freq, new_freq);
 208                         goto rearm;
 209                 }
 210         }
 211
 212         pcpu->floor_freq = new_freq;
 213         pcpu->floor_validate_time = now;
 214
 215         if (pcpu->target_freq == new_freq) {
 216                 trace_cpufreq_interactive_already(data, cpu_load,
 217                                                   pcpu->target_freq, new_freq);
 218                 goto rearm_if_notmax;
 219         }
 220
 221         trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
 222                                          new_freq);
 223         pcpu->target_set_time_in_idle = now_idle;
 224         pcpu->target_set_time = now;
 225
 226         pcpu->target_freq = new_freq;
 227         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 228         cpumask_set_cpu(data, &speedchange_cpumask);
 229         spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 230         wake_up_process(speedchange_task);
 231
 232 rearm_if_notmax:
 233         /*
 234          * Already set max speed and don't see a need to change that,
 235          * wait until next idle to re-evaluate, don't need timer.
 236          */
 237         if (pcpu->target_freq == pcpu->policy->max)
 238                 goto exit;
 239
 240 rearm:
 241         if (!timer_pending(&pcpu->cpu_timer)) {
 242                 /*
 243                  * If already at min, cancel the timer if that CPU goes idle.
 244                  * We don't need to re-evaluate speed until the next idle exit.
 245                  */
 246                 if (pcpu->target_freq == pcpu->policy->min)
 247                         pcpu->timer_idlecancel = 1;
 248
 249                 pcpu->time_in_idle = get_cpu_idle_time_us(
 250                         data, &pcpu->idle_exit_time);
 251                 mod_timer_pinned(&pcpu->cpu_timer,
 252                                  jiffies + usecs_to_jiffies(timer_rate));
 253         }
 254
 255 exit:
 256         return;
 257 }
 258
 259 static void cpufreq_interactive_idle_start(void)
 260 {
 261         struct cpufreq_interactive_cpuinfo *pcpu =
 262                 &per_cpu(cpuinfo, smp_processor_id());
 263         int pending;
 264
 265         if (!pcpu->governor_enabled)
 266                 return;
 267
 268         pending = timer_pending(&pcpu->cpu_timer);
 269
 270         if (pcpu->target_freq != pcpu->policy->min) {
 271 #ifdef CONFIG_SMP
 272                 /*
 273                  * Entering idle while not at lowest speed.  On some
 274                  * platforms this can hold the other CPU(s) at that speed
 275                  * even though the CPU is idle. Set a timer to re-evaluate
 276                  * speed so this idle CPU doesn't hold the other CPUs above
 277                  * min indefinitely.  This should probably be a quirk of
 278                  * the CPUFreq driver.
 279                  */
 280                 if (!pending) {
 281                         pcpu->time_in_idle = get_cpu_idle_time_us(
 282                                 smp_processor_id(), &pcpu->idle_exit_time);
 283                         pcpu->timer_idlecancel = 0;
 284                         mod_timer_pinned(
 285                                 &pcpu->cpu_timer,
 286                                 jiffies + usecs_to_jiffies(timer_rate));
 287                 }
 288 #endif
 289         } else {
 290                 /*
 291                  * If at min speed and entering idle after load has
 292                  * already been evaluated, and a timer has been set just in
 293                  * case the CPU suddenly goes busy, cancel that timer.  The
 294                  * CPU didn't go busy; we'll recheck things upon idle exit.
 295                  */
 296                 if (pending && pcpu->timer_idlecancel) {
 297                         del_timer(&pcpu->cpu_timer);
 298                         pcpu->timer_idlecancel = 0;
 299                 }
 300         }
 301
 302 }
 303
 304 static void cpufreq_interactive_idle_end(void)
 305 {
 306         struct cpufreq_interactive_cpuinfo *pcpu =
 307                 &per_cpu(cpuinfo, smp_processor_id());
 308
 309         if (!pcpu->governor_enabled)
 310                 return;
 311
 312         /* Arm the timer for 1-2 ticks later if not already. */
 313         if (!timer_pending(&pcpu->cpu_timer)) {
 314                 pcpu->time_in_idle =
 315                         get_cpu_idle_time_us(smp_processor_id(),
 316                                              &pcpu->idle_exit_time);
 317                 pcpu->timer_idlecancel = 0;
 318                 mod_timer_pinned(
 319                         &pcpu->cpu_timer,
 320                         jiffies + usecs_to_jiffies(timer_rate));
 321         }
 322
 323 }
 324
 325 static int cpufreq_interactive_speedchange_task(void *data)
 326 {
 327         unsigned int cpu;
 328         cpumask_t tmp_mask;
 329         unsigned long flags;
 330         struct cpufreq_interactive_cpuinfo *pcpu;
 331
 332         while (1) {
 333                 set_current_state(TASK_INTERRUPTIBLE);
 334                 spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 335
 336                 if (cpumask_empty(&speedchange_cpumask)) {
 337                         spin_unlock_irqrestore(&speedchange_cpumask_lock,
 338                                                flags);
 339                         schedule();
 340
 341                         if (kthread_should_stop())
 342                                 break;
 343
 344                         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 345                 }
 346
 347                 set_current_state(TASK_RUNNING);
 348                 tmp_mask = speedchange_cpumask;
 349                 cpumask_clear(&speedchange_cpumask);
 350                 spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 351
 352                 for_each_cpu(cpu, &tmp_mask) {
 353                         unsigned int j;
 354                         unsigned int max_freq = 0;
 355
 356                         pcpu = &per_cpu(cpuinfo, cpu);
 357                         smp_rmb();
 358
 359                         if (!pcpu->governor_enabled)
 360                                 continue;
 361
 362                         for_each_cpu(j, pcpu->policy->cpus) {
 363                                 struct cpufreq_interactive_cpuinfo *pjcpu =
 364                                         &per_cpu(cpuinfo, j);
 365
 366                                 if (pjcpu->target_freq > max_freq)
 367                                         max_freq = pjcpu->target_freq;
 368                         }
 369
 370                         if (max_freq != pcpu->policy->cur)
 371                                 __cpufreq_driver_target(pcpu->policy,
 372                                                         max_freq,
 373                                                         CPUFREQ_RELATION_H);
 374                         trace_cpufreq_interactive_setspeed(cpu,
 375                                                      pcpu->target_freq,
 376                                                      pcpu->policy->cur);
 377                 }
 378         }
 379
 380         return 0;
 381 }
 382
 383 static void cpufreq_interactive_boost(void)
 384 {
 385         int i;
 386         int anyboost = 0;
 387         unsigned long flags;
 388         struct cpufreq_interactive_cpuinfo *pcpu;
 389
 390         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 391
 392         for_each_online_cpu(i) {
 393                 pcpu = &per_cpu(cpuinfo, i);
 394
 395                 if (pcpu->target_freq < hispeed_freq) {
 396                         pcpu->target_freq = hispeed_freq;
 397                         cpumask_set_cpu(i, &speedchange_cpumask);
 398                         pcpu->target_set_time_in_idle =
 399                                 get_cpu_idle_time_us(i, &pcpu->target_set_time);
 400                         pcpu->hispeed_validate_time = pcpu->target_set_time;
 401                         anyboost = 1;
 402                 }
 403
 404                 /*
 405                  * Set floor freq and (re)start timer for when last
 406                  * validated.
 407                  */
 408
 409                 pcpu->floor_freq = hispeed_freq;
 410                 pcpu->floor_validate_time = ktime_to_us(ktime_get());
 411         }
 412
 413         spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 414
 415         if (anyboost)
 416                 wake_up_process(speedchange_task);
 417 }
 418
 419 static ssize_t show_hispeed_freq(struct kobject *kobj,
 420                                  struct attribute *attr, char *buf)
 421 {
 422         return sprintf(buf, "%u\n", hispeed_freq);
 423 }
 424
 425 static ssize_t store_hispeed_freq(struct kobject *kobj,
 426                                   struct attribute *attr, const char *buf,
 427                                   size_t count)
 428 {
 429         int ret;
 430         long unsigned int val;
 431
 432         ret = strict_strtoul(buf, 0, &val);
 433         if (ret < 0)
 434                 return ret;
 435         hispeed_freq = val;
 436         return count;
 437 }
 438
 439 static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644,
 440                 show_hispeed_freq, store_hispeed_freq);
 441
 442
 443 static ssize_t show_go_hispeed_load(struct kobject *kobj,
 444                                      struct attribute *attr, char *buf)
 445 {
 446         return sprintf(buf, "%lu\n", go_hispeed_load);
 447 }
 448
 449 static ssize_t store_go_hispeed_load(struct kobject *kobj,
 450                         struct attribute *attr, const char *buf, size_t count)
 451 {
 452         int ret;
 453         unsigned long val;
 454
 455         ret = strict_strtoul(buf, 0, &val);
 456         if (ret < 0)
 457                 return ret;
 458         go_hispeed_load = val;
 459         return count;
 460 }
 461
 462 static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644,
 463                 show_go_hispeed_load, store_go_hispeed_load);
 464
 465 static ssize_t show_min_sample_time(struct kobject *kobj,
 466                                 struct attribute *attr, char *buf)
 467 {
 468         return sprintf(buf, "%lu\n", min_sample_time);
 469 }
 470
 471 static ssize_t store_min_sample_time(struct kobject *kobj,
 472                         struct attribute *attr, const char *buf, size_t count)
 473 {
 474         int ret;
 475         unsigned long val;
 476
 477         ret = strict_strtoul(buf, 0, &val);
 478         if (ret < 0)
 479                 return ret;
 480         min_sample_time = val;
 481         return count;
 482 }
 483
 484 static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
 485                 show_min_sample_time, store_min_sample_time);
 486
 487 static ssize_t show_above_hispeed_delay(struct kobject *kobj,
 488                                         struct attribute *attr, char *buf)
 489 {
 490         return sprintf(buf, "%lu\n", above_hispeed_delay_val);
 491 }
 492
 493 static ssize_t store_above_hispeed_delay(struct kobject *kobj,
 494                                          struct attribute *attr,
 495                                          const char *buf, size_t count)
 496 {
 497         int ret;
 498         unsigned long val;
 499
 500         ret = strict_strtoul(buf, 0, &val);
 501         if (ret < 0)
 502                 return ret;
 503         above_hispeed_delay_val = val;
 504         return count;
 505 }
 506
 507 define_one_global_rw(above_hispeed_delay);
 508
 509 static ssize_t show_timer_rate(struct kobject *kobj,
 510                         struct attribute *attr, char *buf)
 511 {
 512         return sprintf(buf, "%lu\n", timer_rate);
 513 }
 514
 515 static ssize_t store_timer_rate(struct kobject *kobj,
 516                         struct attribute *attr, const char *buf, size_t count)
 517 {
 518         int ret;
 519         unsigned long val;
 520
 521         ret = strict_strtoul(buf, 0, &val);
 522         if (ret < 0)
 523                 return ret;
 524         timer_rate = val;
 525         return count;
 526 }
 527
 528 static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
 529                 show_timer_rate, store_timer_rate);
 530
 531 static ssize_t show_boost(struct kobject *kobj, struct attribute *attr,
 532                           char *buf)
 533 {
 534         return sprintf(buf, "%d\n", boost_val);
 535 }
 536
 537 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 538                            const char *buf, size_t count)
 539 {
 540         int ret;
 541         unsigned long val;
 542
 543         ret = kstrtoul(buf, 0, &val);
 544         if (ret < 0)
 545                 return ret;
 546
 547         boost_val = val;
 548
 549         if (boost_val) {
 550                 trace_cpufreq_interactive_boost("on");
 551                 cpufreq_interactive_boost();
 552         } else {
 553                 trace_cpufreq_interactive_unboost("off");
 554         }
 555
 556         return count;
 557 }
 558
 559 define_one_global_rw(boost);
 560
 561 static ssize_t store_boostpulse(struct kobject *kobj, struct attribute *attr,
 562                                 const char *buf, size_t count)
 563 {
 564         int ret;
 565         unsigned long val;
 566
 567         ret = kstrtoul(buf, 0, &val);
 568         if (ret < 0)
 569                 return ret;
 570
 571         trace_cpufreq_interactive_boost("pulse");
 572         cpufreq_interactive_boost();
 573         return count;
 574 }
 575
 576 static struct global_attr boostpulse =
 577         __ATTR(boostpulse, 0200, NULL, store_boostpulse);
 578
 579 static struct attribute *interactive_attributes[] = {
 580         &hispeed_freq_attr.attr,
 581         &go_hispeed_load_attr.attr,
 582         &above_hispeed_delay.attr,
 583         &min_sample_time_attr.attr,
 584         &timer_rate_attr.attr,
 585         &boost.attr,
 586         &boostpulse.attr,
 587         NULL,
 588 };
 589
 590 static struct attribute_group interactive_attr_group = {
 591         .attrs = interactive_attributes,
 592         .name = "interactive",
 593 };
 594
 595 static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
 596                                              unsigned long val,
 597                                              void *data)
 598 {
 599         switch (val) {
 600         case IDLE_START:
 601                 cpufreq_interactive_idle_start();
 602                 break;
 603         case IDLE_END:
 604                 cpufreq_interactive_idle_end();
 605                 break;
 606         }
 607
 608         return 0;
 609 }
 610
 611 static struct notifier_block cpufreq_interactive_idle_nb = {
 612         .notifier_call = cpufreq_interactive_idle_notifier,
 613 };
 614
 615 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 616                 unsigned int event)
 617 {
 618         int rc;
 619         unsigned int j;
 620         struct cpufreq_interactive_cpuinfo *pcpu;
 621         struct cpufreq_frequency_table *freq_table;
 622
 623         switch (event) {
 624         case CPUFREQ_GOV_START:
 625                 if (!cpu_online(policy->cpu))
 626                         return -EINVAL;
 627
 628                 freq_table =
 629                         cpufreq_frequency_get_table(policy->cpu);
 630                 if (!hispeed_freq)
 631                         hispeed_freq = policy->max;
 632
 633                 for_each_cpu(j, policy->cpus) {
 634                         pcpu = &per_cpu(cpuinfo, j);
 635                         pcpu->policy = policy;
 636                         pcpu->target_freq = policy->cur;
 637                         pcpu->freq_table = freq_table;
 638                         pcpu->target_set_time_in_idle =
 639                                 get_cpu_idle_time_us(j,
 640                                              &pcpu->target_set_time);
 641                         pcpu->floor_freq = pcpu->target_freq;
 642                         pcpu->floor_validate_time =
 643                                 pcpu->target_set_time;
 644                         pcpu->hispeed_validate_time =
 645                                 pcpu->target_set_time;
 646                         pcpu->governor_enabled = 1;
 647                         smp_wmb();
 648                         pcpu->cpu_timer.expires =
 649                                 jiffies + usecs_to_jiffies(timer_rate);
 650                         add_timer_on(&pcpu->cpu_timer, j);
 651                 }
 652
 653                 /*
 654                  * Do not register the idle hook and create sysfs
 655                  * entries if we have already done so.
 656                  */
 657                 if (atomic_inc_return(&active_count) > 1)
 658                         return 0;
 659
 660                 rc = sysfs_create_group(cpufreq_global_kobject,
 661                                 &interactive_attr_group);
 662                 if (rc)
 663                         return rc;
 664
 665                 idle_notifier_register(&cpufreq_interactive_idle_nb);
 666                 break;
 667
 668         case CPUFREQ_GOV_STOP:
 669                 for_each_cpu(j, policy->cpus) {
 670                         pcpu = &per_cpu(cpuinfo, j);
 671                         pcpu->governor_enabled = 0;
 672                         smp_wmb();
 673                         del_timer_sync(&pcpu->cpu_timer);
 674                 }
 675
 676                 if (atomic_dec_return(&active_count) > 0)
 677                         return 0;
 678
 679                 idle_notifier_unregister(&cpufreq_interactive_idle_nb);
 680                 sysfs_remove_group(cpufreq_global_kobject,
 681                                 &interactive_attr_group);
 682
 683                 break;
 684
 685         case CPUFREQ_GOV_LIMITS:
 686                 if (policy->max < policy->cur)
 687                         __cpufreq_driver_target(policy,
 688                                         policy->max, CPUFREQ_RELATION_H);
 689                 else if (policy->min > policy->cur)
 690                         __cpufreq_driver_target(policy,
 691                                         policy->min, CPUFREQ_RELATION_L);
 692                 break;
 693         }
 694         return 0;
 695 }
 696
 697 static int __init cpufreq_interactive_init(void)
 698 {
 699         unsigned int i;
 700         struct cpufreq_interactive_cpuinfo *pcpu;
 701         struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 702
 703         go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
 704         min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
 705         above_hispeed_delay_val = DEFAULT_ABOVE_HISPEED_DELAY;
 706         timer_rate = DEFAULT_TIMER_RATE;
 707
 708         /* Initalize per-cpu timers */
 709         for_each_possible_cpu(i) {
 710                 pcpu = &per_cpu(cpuinfo, i);
 711                 init_timer(&pcpu->cpu_timer);
 712                 pcpu->cpu_timer.function = cpufreq_interactive_timer;
 713                 pcpu->cpu_timer.data = i;
 714         }
 715
 716         spin_lock_init(&speedchange_cpumask_lock);
 717         speedchange_task =
 718                 kthread_create(cpufreq_interactive_speedchange_task, NULL,
 719                                "cfinteractive");
 720         if (IS_ERR(speedchange_task))
 721                 return PTR_ERR(speedchange_task);
 722
 723         sched_setscheduler_nocheck(speedchange_task, SCHED_FIFO, &param);
 724         get_task_struct(speedchange_task);
 725
 726         /* NB: wake up so the thread does not look hung to the freezer */
 727         wake_up_process(speedchange_task);
 728
 729         return cpufreq_register_governor(&cpufreq_gov_interactive);
 730 }
 731
 732 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 733 fs_initcall(cpufreq_interactive_init);
 734 #else
 735 module_init(cpufreq_interactive_init);
 736 #endif
 737
 738 static void __exit cpufreq_interactive_exit(void)
 739 {
 740         cpufreq_unregister_governor(&cpufreq_gov_interactive);
 741         kthread_stop(speedchange_task);
 742         put_task_struct(speedchange_task);
 743 }
 744
 745 module_exit(cpufreq_interactive_exit);
 746
 747 MODULE_AUTHOR("Mike Chan <mike@android.com>");
 748 MODULE_DESCRIPTION("'cpufreq_interactive' - A cpufreq governor for "
 749         "Latency sensitive workloads");
 750 MODULE_LICENSE("GPL");