drivers/cpufreq/cpufreq_interactive.c

   1 /*
   2  * drivers/cpufreq/cpufreq_interactive.c
   3  *
   4  * Copyright (C) 2010 Google, Inc.
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * Author: Mike Chan (mike@android.com)
  16  *
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/cpumask.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/module.h>
  23 #include <linux/moduleparam.h>
  24 #include <linux/mutex.h>
  25 #include <linux/sched.h>
  26 #include <linux/sched/rt.h>
  27 #include <linux/tick.h>
  28 #include <linux/time.h>
  29 #include <linux/timer.h>
  30 #include <linux/workqueue.h>
  31 #include <linux/kthread.h>
  32 #include <linux/mutex.h>
  33 #include <linux/slab.h>
  34 #include <asm/cputime.h>
  35
  36 #define CREATE_TRACE_POINTS
  37 #include <trace/events/cpufreq_interactive.h>
  38
  39 static atomic_t active_count = ATOMIC_INIT(0);
  40
  41 struct cpufreq_interactive_cpuinfo {
  42         struct timer_list cpu_timer;
  43         int timer_idlecancel;
  44         u64 time_in_idle;
  45         u64 time_in_idle_timestamp;
  46         u64 target_set_time;
  47         u64 target_set_time_in_idle;
  48         struct cpufreq_policy *policy;
  49         struct cpufreq_frequency_table *freq_table;
  50         unsigned int target_freq;
  51         unsigned int floor_freq;
  52         u64 floor_validate_time;
  53         u64 hispeed_validate_time;
  54         int governor_enabled;
  55 };
  56
  57 static DEFINE_PER_CPU(struct cpufreq_interactive_cpuinfo, cpuinfo);
  58
  59 /* realtime thread handles frequency scaling */
  60 static struct task_struct *speedchange_task;
  61 static cpumask_t speedchange_cpumask;
  62 static spinlock_t speedchange_cpumask_lock;
  63
  64 /* Hi speed to bump to from lo speed when load burst (default max) */
  65 static unsigned int hispeed_freq;
  66
  67 /* Go to hi speed when CPU load at or above this value. */
  68 #define DEFAULT_GO_HISPEED_LOAD 85
  69 static unsigned long go_hispeed_load;
  70
  71 /* Target load.  Lower values result in higher CPU speeds. */
  72 #define DEFAULT_TARGET_LOAD 90
  73 static unsigned long target_load = DEFAULT_TARGET_LOAD;
  74
  75 /*
  76  * The minimum amount of time to spend at a frequency before we can ramp down.
  77  */
  78 #define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
  79 static unsigned long min_sample_time;
  80
  81 /*
  82  * The sample rate of the timer used to increase frequency
  83  */
  84 #define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC)
  85 static unsigned long timer_rate;
  86
  87 /*
  88  * Wait this long before raising speed above hispeed, by default a single
  89  * timer interval.
  90  */
  91 #define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE
  92 static unsigned long above_hispeed_delay_val;
  93
  94 /*
  95  * Non-zero means longer-term speed boost active.
  96  */
  97
  98 static int boost_val;
  99
 100 static bool governidle;
 101 module_param(governidle, bool, S_IWUSR | S_IRUGO);
 102 MODULE_PARM_DESC(governidle,
 103         "Set to 1 to wake up CPUs from idle to reduce speed (default 0)");
 104
 105 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 106                 unsigned int event);
 107
 108 #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 109 static
 110 #endif
 111 struct cpufreq_governor cpufreq_gov_interactive = {
 112         .name = "interactive",
 113         .governor = cpufreq_governor_interactive,
 114         .max_transition_latency = 10000000,
 115         .owner = THIS_MODULE,
 116 };
 117
 118 static void cpufreq_interactive_timer_resched(
 119         struct cpufreq_interactive_cpuinfo *pcpu)
 120 {
 121         mod_timer_pinned(&pcpu->cpu_timer,
 122                          jiffies + usecs_to_jiffies(timer_rate));
 123         pcpu->time_in_idle =
 124                 get_cpu_idle_time_us(smp_processor_id(),
 125                                      &pcpu->time_in_idle_timestamp);
 126 }
 127
 128 static void cpufreq_interactive_timer(unsigned long data)
 129 {
 130         u64 now;
 131         unsigned int delta_idle;
 132         unsigned int delta_time;
 133         int cpu_load;
 134         int load_since_change;
 135         struct cpufreq_interactive_cpuinfo *pcpu =
 136                 &per_cpu(cpuinfo, data);
 137         u64 now_idle;
 138         unsigned int new_freq;
 139         unsigned int index;
 140         unsigned long flags;
 141
 142         smp_rmb();
 143
 144         if (!pcpu->governor_enabled)
 145                 goto exit;
 146
 147         now_idle = get_cpu_idle_time_us(data, &now);
 148         delta_idle = (unsigned int)(now_idle - pcpu->time_in_idle);
 149         delta_time = (unsigned int)(now - pcpu->time_in_idle_timestamp);
 150
 151         /*
 152          * If timer ran less than 1ms after short-term sample started, retry.
 153          */
 154         if (delta_time < 1000)
 155                 goto rearm;
 156
 157         if (delta_idle > delta_time)
 158                 cpu_load = 0;
 159         else
 160                 cpu_load = 100 * (delta_time - delta_idle) / delta_time;
 161
 162         delta_idle = (unsigned int)(now_idle - pcpu->target_set_time_in_idle);
 163         delta_time = (unsigned int)(now - pcpu->target_set_time);
 164
 165         if ((delta_time == 0) || (delta_idle > delta_time))
 166                 load_since_change = 0;
 167         else
 168                 load_since_change =
 169                         100 * (delta_time - delta_idle) / delta_time;
 170
 171         /*
 172          * Choose greater of short-term load (since last idle timer
 173          * started or timer function re-armed itself) or long-term load
 174          * (since last frequency change).
 175          */
 176         if (load_since_change > cpu_load)
 177                 cpu_load = load_since_change;
 178
 179         if (cpu_load >= go_hispeed_load || boost_val) {
 180                 if (pcpu->target_freq < hispeed_freq &&
 181                     hispeed_freq < pcpu->policy->max) {
 182                         new_freq = hispeed_freq;
 183                 } else {
 184                         new_freq = pcpu->policy->cur * cpu_load / target_load;
 185
 186                         if (new_freq < hispeed_freq)
 187                                 new_freq = hispeed_freq;
 188
 189                         if (pcpu->target_freq == hispeed_freq &&
 190                             new_freq > hispeed_freq &&
 191                             now - pcpu->hispeed_validate_time
 192                             < above_hispeed_delay_val) {
 193                                 trace_cpufreq_interactive_notyet(
 194                                         data, cpu_load, pcpu->target_freq,
 195                                         pcpu->policy->cur, new_freq);
 196                                 goto rearm;
 197                         }
 198                 }
 199         } else {
 200                 new_freq = pcpu->policy->cur * cpu_load / target_load;
 201         }
 202
 203         if (new_freq <= hispeed_freq)
 204                 pcpu->hispeed_validate_time = now;
 205
 206         if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
 207                                            new_freq, CPUFREQ_RELATION_L,
 208                                            &index)) {
 209                 pr_warn_once("timer %d: cpufreq_frequency_table_target error\n",
 210                              (int) data);
 211                 goto rearm;
 212         }
 213
 214         new_freq = pcpu->freq_table[index].frequency;
 215
 216         /*
 217          * Do not scale below floor_freq unless we have been at or above the
 218          * floor frequency for the minimum sample time since last validated.
 219          */
 220         if (new_freq < pcpu->floor_freq) {
 221                 if (now - pcpu->floor_validate_time < min_sample_time) {
 222                         trace_cpufreq_interactive_notyet(
 223                                 data, cpu_load, pcpu->target_freq,
 224                                 pcpu->policy->cur, new_freq);
 225                         goto rearm;
 226                 }
 227         }
 228
 229         pcpu->floor_freq = new_freq;
 230         pcpu->floor_validate_time = now;
 231
 232         if (pcpu->target_freq == new_freq) {
 233                 trace_cpufreq_interactive_already(
 234                         data, cpu_load, pcpu->target_freq,
 235                         pcpu->policy->cur, new_freq);
 236                 goto rearm_if_notmax;
 237         }
 238
 239         trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
 240                                          pcpu->policy->cur, new_freq);
 241         pcpu->target_set_time_in_idle = now_idle;
 242         pcpu->target_set_time = now;
 243
 244         pcpu->target_freq = new_freq;
 245         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 246         cpumask_set_cpu(data, &speedchange_cpumask);
 247         spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 248         wake_up_process(speedchange_task);
 249
 250 rearm_if_notmax:
 251         /*
 252          * Already set max speed and don't see a need to change that,
 253          * wait until next idle to re-evaluate, don't need timer.
 254          */
 255         if (pcpu->target_freq == pcpu->policy->max)
 256                 goto exit;
 257
 258 rearm:
 259         if (!timer_pending(&pcpu->cpu_timer)) {
 260                 /*
 261                  * If governing speed in idle and already at min, cancel the
 262                  * timer if that CPU goes idle.  We don't need to re-evaluate
 263                  * speed until the next idle exit.
 264                  */
 265                 if (governidle && pcpu->target_freq == pcpu->policy->min)
 266                         pcpu->timer_idlecancel = 1;
 267
 268                 cpufreq_interactive_timer_resched(pcpu);
 269         }
 270
 271 exit:
 272         return;
 273 }
 274
 275 static void cpufreq_interactive_idle_start(void)
 276 {
 277         struct cpufreq_interactive_cpuinfo *pcpu =
 278                 &per_cpu(cpuinfo, smp_processor_id());
 279         int pending;
 280
 281         if (!pcpu->governor_enabled)
 282                 return;
 283
 284         pending = timer_pending(&pcpu->cpu_timer);
 285
 286         if (pcpu->target_freq != pcpu->policy->min) {
 287                 /*
 288                  * Entering idle while not at lowest speed.  On some
 289                  * platforms this can hold the other CPU(s) at that speed
 290                  * even though the CPU is idle. Set a timer to re-evaluate
 291                  * speed so this idle CPU doesn't hold the other CPUs above
 292                  * min indefinitely.  This should probably be a quirk of
 293                  * the CPUFreq driver.
 294                  */
 295                 if (!pending) {
 296                         pcpu->timer_idlecancel = 0;
 297                         cpufreq_interactive_timer_resched(pcpu);
 298                 }
 299         } else if (governidle) {
 300                 /*
 301                  * If at min speed and entering idle after load has
 302                  * already been evaluated, and a timer has been set just in
 303                  * case the CPU suddenly goes busy, cancel that timer.  The
 304                  * CPU didn't go busy; we'll recheck things upon idle exit.
 305                  */
 306                 if (pending && pcpu->timer_idlecancel) {
 307                         del_timer(&pcpu->cpu_timer);
 308                         pcpu->timer_idlecancel = 0;
 309                 }
 310         }
 311
 312 }
 313
 314 static void cpufreq_interactive_idle_end(void)
 315 {
 316         struct cpufreq_interactive_cpuinfo *pcpu =
 317                 &per_cpu(cpuinfo, smp_processor_id());
 318
 319         if (!pcpu->governor_enabled)
 320                 return;
 321
 322         /* Arm the timer for 1-2 ticks later if not already. */
 323         if (!timer_pending(&pcpu->cpu_timer)) {
 324                 pcpu->timer_idlecancel = 0;
 325                 cpufreq_interactive_timer_resched(pcpu);
 326         } else if (!governidle &&
 327                    time_after_eq(jiffies, pcpu->cpu_timer.expires)) {
 328                 del_timer(&pcpu->cpu_timer);
 329                 cpufreq_interactive_timer(smp_processor_id());
 330         }
 331 }
 332
 333 static int cpufreq_interactive_speedchange_task(void *data)
 334 {
 335         unsigned int cpu;
 336         cpumask_t tmp_mask;
 337         unsigned long flags;
 338         struct cpufreq_interactive_cpuinfo *pcpu;
 339
 340         while (1) {
 341                 set_current_state(TASK_INTERRUPTIBLE);
 342                 spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 343
 344                 if (cpumask_empty(&speedchange_cpumask)) {
 345                         spin_unlock_irqrestore(&speedchange_cpumask_lock,
 346                                                flags);
 347                         schedule();
 348
 349                         if (kthread_should_stop())
 350                                 break;
 351
 352                         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 353                 }
 354
 355                 set_current_state(TASK_RUNNING);
 356                 tmp_mask = speedchange_cpumask;
 357                 cpumask_clear(&speedchange_cpumask);
 358                 spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 359
 360                 for_each_cpu(cpu, &tmp_mask) {
 361                         unsigned int j;
 362                         unsigned int max_freq = 0;
 363
 364                         pcpu = &per_cpu(cpuinfo, cpu);
 365                         smp_rmb();
 366
 367                         if (!pcpu->governor_enabled)
 368                                 continue;
 369
 370                         for_each_cpu(j, pcpu->policy->cpus) {
 371                                 struct cpufreq_interactive_cpuinfo *pjcpu =
 372                                         &per_cpu(cpuinfo, j);
 373
 374                                 if (pjcpu->target_freq > max_freq)
 375                                         max_freq = pjcpu->target_freq;
 376                         }
 377
 378                         if (max_freq != pcpu->policy->cur)
 379                                 __cpufreq_driver_target(pcpu->policy,
 380                                                         max_freq,
 381                                                         CPUFREQ_RELATION_H);
 382                         trace_cpufreq_interactive_setspeed(cpu,
 383                                                      pcpu->target_freq,
 384                                                      pcpu->policy->cur);
 385                 }
 386         }
 387
 388         return 0;
 389 }
 390
 391 static void cpufreq_interactive_boost(void)
 392 {
 393         int i;
 394         int anyboost = 0;
 395         unsigned long flags;
 396         struct cpufreq_interactive_cpuinfo *pcpu;
 397
 398         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 399
 400         for_each_online_cpu(i) {
 401                 pcpu = &per_cpu(cpuinfo, i);
 402
 403                 if (pcpu->target_freq < hispeed_freq) {
 404                         pcpu->target_freq = hispeed_freq;
 405                         cpumask_set_cpu(i, &speedchange_cpumask);
 406                         pcpu->target_set_time_in_idle =
 407                                 get_cpu_idle_time_us(i, &pcpu->target_set_time);
 408                         pcpu->hispeed_validate_time = pcpu->target_set_time;
 409                         anyboost = 1;
 410                 }
 411
 412                 /*
 413                  * Set floor freq and (re)start timer for when last
 414                  * validated.
 415                  */
 416
 417                 pcpu->floor_freq = hispeed_freq;
 418                 pcpu->floor_validate_time = ktime_to_us(ktime_get());
 419         }
 420
 421         spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 422
 423         if (anyboost)
 424                 wake_up_process(speedchange_task);
 425 }
 426
 427 static ssize_t show_target_load(
 428         struct kobject *kobj, struct attribute *attr, char *buf)
 429 {
 430         return sprintf(buf, "%lu\n", target_load);
 431 }
 432
 433 static ssize_t store_target_load(
 434         struct kobject *kobj, struct attribute *attr, const char *buf,
 435         size_t count)
 436 {
 437         int ret;
 438         unsigned long val;
 439
 440         ret = strict_strtoul(buf, 0, &val);
 441         if (ret < 0)
 442                 return ret;
 443         target_load = val;
 444         return count;
 445 }
 446
 447 static struct global_attr target_load_attr =
 448         __ATTR(target_load, S_IRUGO | S_IWUSR,
 449                 show_target_load, store_target_load);
 450
 451 static ssize_t show_hispeed_freq(struct kobject *kobj,
 452                                  struct attribute *attr, char *buf)
 453 {
 454         return sprintf(buf, "%u\n", hispeed_freq);
 455 }
 456
 457 static ssize_t store_hispeed_freq(struct kobject *kobj,
 458                                   struct attribute *attr, const char *buf,
 459                                   size_t count)
 460 {
 461         int ret;
 462         long unsigned int val;
 463
 464         ret = strict_strtoul(buf, 0, &val);
 465         if (ret < 0)
 466                 return ret;
 467         hispeed_freq = val;
 468         return count;
 469 }
 470
 471 static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644,
 472                 show_hispeed_freq, store_hispeed_freq);
 473
 474
 475 static ssize_t show_go_hispeed_load(struct kobject *kobj,
 476                                      struct attribute *attr, char *buf)
 477 {
 478         return sprintf(buf, "%lu\n", go_hispeed_load);
 479 }
 480
 481 static ssize_t store_go_hispeed_load(struct kobject *kobj,
 482                         struct attribute *attr, const char *buf, size_t count)
 483 {
 484         int ret;
 485         unsigned long val;
 486
 487         ret = strict_strtoul(buf, 0, &val);
 488         if (ret < 0)
 489                 return ret;
 490         go_hispeed_load = val;
 491         return count;
 492 }
 493
 494 static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644,
 495                 show_go_hispeed_load, store_go_hispeed_load);
 496
 497 static ssize_t show_min_sample_time(struct kobject *kobj,
 498                                 struct attribute *attr, char *buf)
 499 {
 500         return sprintf(buf, "%lu\n", min_sample_time);
 501 }
 502
 503 static ssize_t store_min_sample_time(struct kobject *kobj,
 504                         struct attribute *attr, const char *buf, size_t count)
 505 {
 506         int ret;
 507         unsigned long val;
 508
 509         ret = strict_strtoul(buf, 0, &val);
 510         if (ret < 0)
 511                 return ret;
 512         min_sample_time = val;
 513         return count;
 514 }
 515
 516 static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
 517                 show_min_sample_time, store_min_sample_time);
 518
 519 static ssize_t show_above_hispeed_delay(struct kobject *kobj,
 520                                         struct attribute *attr, char *buf)
 521 {
 522         return sprintf(buf, "%lu\n", above_hispeed_delay_val);
 523 }
 524
 525 static ssize_t store_above_hispeed_delay(struct kobject *kobj,
 526                                          struct attribute *attr,
 527                                          const char *buf, size_t count)
 528 {
 529         int ret;
 530         unsigned long val;
 531
 532         ret = strict_strtoul(buf, 0, &val);
 533         if (ret < 0)
 534                 return ret;
 535         above_hispeed_delay_val = val;
 536         return count;
 537 }
 538
 539 define_one_global_rw(above_hispeed_delay);
 540
 541 static ssize_t show_timer_rate(struct kobject *kobj,
 542                         struct attribute *attr, char *buf)
 543 {
 544         return sprintf(buf, "%lu\n", timer_rate);
 545 }
 546
 547 static ssize_t store_timer_rate(struct kobject *kobj,
 548                         struct attribute *attr, const char *buf, size_t count)
 549 {
 550         int ret;
 551         unsigned long val;
 552
 553         ret = strict_strtoul(buf, 0, &val);
 554         if (ret < 0)
 555                 return ret;
 556         timer_rate = val;
 557         return count;
 558 }
 559
 560 static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
 561                 show_timer_rate, store_timer_rate);
 562
 563 static ssize_t show_boost(struct kobject *kobj, struct attribute *attr,
 564                           char *buf)
 565 {
 566         return sprintf(buf, "%d\n", boost_val);
 567 }
 568
 569 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 570                            const char *buf, size_t count)
 571 {
 572         int ret;
 573         unsigned long val;
 574
 575         ret = kstrtoul(buf, 0, &val);
 576         if (ret < 0)
 577                 return ret;
 578
 579         boost_val = val;
 580
 581         if (boost_val) {
 582                 trace_cpufreq_interactive_boost("on");
 583                 cpufreq_interactive_boost();
 584         } else {
 585                 trace_cpufreq_interactive_unboost("off");
 586         }
 587
 588         return count;
 589 }
 590
 591 define_one_global_rw(boost);
 592
 593 static ssize_t store_boostpulse(struct kobject *kobj, struct attribute *attr,
 594                                 const char *buf, size_t count)
 595 {
 596         int ret;
 597         unsigned long val;
 598
 599         ret = kstrtoul(buf, 0, &val);
 600         if (ret < 0)
 601                 return ret;
 602
 603         trace_cpufreq_interactive_boost("pulse");
 604         cpufreq_interactive_boost();
 605         return count;
 606 }
 607
 608 static struct global_attr boostpulse =
 609         __ATTR(boostpulse, 0200, NULL, store_boostpulse);
 610
 611 static struct attribute *interactive_attributes[] = {
 612         &target_load_attr.attr,
 613         &hispeed_freq_attr.attr,
 614         &go_hispeed_load_attr.attr,
 615         &above_hispeed_delay.attr,
 616         &min_sample_time_attr.attr,
 617         &timer_rate_attr.attr,
 618         &boost.attr,
 619         &boostpulse.attr,
 620         NULL,
 621 };
 622
 623 static struct attribute_group interactive_attr_group = {
 624         .attrs = interactive_attributes,
 625         .name = "interactive",
 626 };
 627
 628 static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
 629                                              unsigned long val,
 630                                              void *data)
 631 {
 632         switch (val) {
 633         case IDLE_START:
 634                 cpufreq_interactive_idle_start();
 635                 break;
 636         case IDLE_END:
 637                 cpufreq_interactive_idle_end();
 638                 break;
 639         }
 640
 641         return 0;
 642 }
 643
 644 static struct notifier_block cpufreq_interactive_idle_nb = {
 645         .notifier_call = cpufreq_interactive_idle_notifier,
 646 };
 647
 648 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 649                 unsigned int event)
 650 {
 651         int rc;
 652         unsigned int j;
 653         struct cpufreq_interactive_cpuinfo *pcpu;
 654         struct cpufreq_frequency_table *freq_table;
 655
 656         switch (event) {
 657         case CPUFREQ_GOV_START:
 658                 if (!cpu_online(policy->cpu))
 659                         return -EINVAL;
 660
 661                 freq_table =
 662                         cpufreq_frequency_get_table(policy->cpu);
 663                 if (!hispeed_freq)
 664                         hispeed_freq = policy->max;
 665
 666                 for_each_cpu(j, policy->cpus) {
 667                         pcpu = &per_cpu(cpuinfo, j);
 668                         pcpu->policy = policy;
 669                         pcpu->target_freq = policy->cur;
 670                         pcpu->freq_table = freq_table;
 671                         pcpu->target_set_time_in_idle =
 672                                 get_cpu_idle_time_us(j,
 673                                              &pcpu->target_set_time);
 674                         pcpu->floor_freq = pcpu->target_freq;
 675                         pcpu->floor_validate_time =
 676                                 pcpu->target_set_time;
 677                         pcpu->hispeed_validate_time =
 678                                 pcpu->target_set_time;
 679                         pcpu->governor_enabled = 1;
 680                         smp_wmb();
 681                         pcpu->cpu_timer.expires =
 682                                 jiffies + usecs_to_jiffies(timer_rate);
 683                         add_timer_on(&pcpu->cpu_timer, j);
 684                 }
 685
 686                 /*
 687                  * Do not register the idle hook and create sysfs
 688                  * entries if we have already done so.
 689                  */
 690                 if (atomic_inc_return(&active_count) > 1)
 691                         return 0;
 692
 693                 rc = sysfs_create_group(cpufreq_global_kobject,
 694                                 &interactive_attr_group);
 695                 if (rc)
 696                         return rc;
 697
 698                 idle_notifier_register(&cpufreq_interactive_idle_nb);
 699                 break;
 700
 701         case CPUFREQ_GOV_STOP:
 702                 for_each_cpu(j, policy->cpus) {
 703                         pcpu = &per_cpu(cpuinfo, j);
 704                         pcpu->governor_enabled = 0;
 705                         smp_wmb();
 706                         del_timer_sync(&pcpu->cpu_timer);
 707                 }
 708
 709                 if (atomic_dec_return(&active_count) > 0)
 710                         return 0;
 711
 712                 idle_notifier_unregister(&cpufreq_interactive_idle_nb);
 713                 sysfs_remove_group(cpufreq_global_kobject,
 714                                 &interactive_attr_group);
 715
 716                 break;
 717
 718         case CPUFREQ_GOV_LIMITS:
 719                 if (policy->max < policy->cur)
 720                         __cpufreq_driver_target(policy,
 721                                         policy->max, CPUFREQ_RELATION_H);
 722                 else if (policy->min > policy->cur)
 723                         __cpufreq_driver_target(policy,
 724                                         policy->min, CPUFREQ_RELATION_L);
 725                 break;
 726         }
 727         return 0;
 728 }
 729
 730 static int __init cpufreq_interactive_init(void)
 731 {
 732         unsigned int i;
 733         struct cpufreq_interactive_cpuinfo *pcpu;
 734         struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 735
 736         go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
 737         min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
 738         above_hispeed_delay_val = DEFAULT_ABOVE_HISPEED_DELAY;
 739         timer_rate = DEFAULT_TIMER_RATE;
 740
 741         /* Initalize per-cpu timers */
 742         for_each_possible_cpu(i) {
 743                 pcpu = &per_cpu(cpuinfo, i);
 744                 if (governidle)
 745                         init_timer(&pcpu->cpu_timer);
 746                 else
 747                         init_timer_deferrable(&pcpu->cpu_timer);
 748                 pcpu->cpu_timer.function = cpufreq_interactive_timer;
 749                 pcpu->cpu_timer.data = i;
 750         }
 751
 752         spin_lock_init(&speedchange_cpumask_lock);
 753         speedchange_task =
 754                 kthread_create(cpufreq_interactive_speedchange_task, NULL,
 755                                "cfinteractive");
 756         if (IS_ERR(speedchange_task))
 757                 return PTR_ERR(speedchange_task);
 758
 759         sched_setscheduler_nocheck(speedchange_task, SCHED_FIFO, &param);
 760         get_task_struct(speedchange_task);
 761
 762         /* NB: wake up so the thread does not look hung to the freezer */
 763         wake_up_process(speedchange_task);
 764
 765         return cpufreq_register_governor(&cpufreq_gov_interactive);
 766 }
 767
 768 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 769 fs_initcall(cpufreq_interactive_init);
 770 #else
 771 module_init(cpufreq_interactive_init);
 772 #endif
 773
 774 static void __exit cpufreq_interactive_exit(void)
 775 {
 776         cpufreq_unregister_governor(&cpufreq_gov_interactive);
 777         kthread_stop(speedchange_task);
 778         put_task_struct(speedchange_task);
 779 }
 780
 781 module_exit(cpufreq_interactive_exit);
 782
 783 MODULE_AUTHOR("Mike Chan <mike@android.com>");
 784 MODULE_DESCRIPTION("'cpufreq_interactive' - A cpufreq governor for "
 785         "Latency sensitive workloads");
 786 MODULE_LICENSE("GPL");