drivers/cpufreq/cpufreq_interactive.c

   1 /*
   2  * drivers/cpufreq/cpufreq_interactive.c
   3  *
   4  * Copyright (C) 2010 Google, Inc.
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * Author: Mike Chan (mike@android.com)
  16  *
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/cpumask.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/module.h>
  23 #include <linux/moduleparam.h>
  24 #include <linux/mutex.h>
  25 #include <linux/sched.h>
  26 #include <linux/sched/rt.h>
  27 #include <linux/tick.h>
  28 #include <linux/time.h>
  29 #include <linux/timer.h>
  30 #include <linux/workqueue.h>
  31 #include <linux/kthread.h>
  32 #include <linux/mutex.h>
  33 #include <linux/slab.h>
  34 #include <asm/cputime.h>
  35
  36 #define CREATE_TRACE_POINTS
  37 #include <trace/events/cpufreq_interactive.h>
  38
  39 static atomic_t active_count = ATOMIC_INIT(0);
  40
  41 struct cpufreq_interactive_cpuinfo {
  42         struct timer_list cpu_timer;
  43         int timer_idlecancel;
  44         u64 time_in_idle;
  45         u64 time_in_idle_timestamp;
  46         struct cpufreq_policy *policy;
  47         struct cpufreq_frequency_table *freq_table;
  48         unsigned int target_freq;
  49         unsigned int floor_freq;
  50         u64 floor_validate_time;
  51         u64 hispeed_validate_time;
  52         int governor_enabled;
  53 };
  54
  55 static DEFINE_PER_CPU(struct cpufreq_interactive_cpuinfo, cpuinfo);
  56
  57 /* realtime thread handles frequency scaling */
  58 static struct task_struct *speedchange_task;
  59 static cpumask_t speedchange_cpumask;
  60 static spinlock_t speedchange_cpumask_lock;
  61
  62 /* Hi speed to bump to from lo speed when load burst (default max) */
  63 static unsigned int hispeed_freq;
  64
  65 /* Go to hi speed when CPU load at or above this value. */
  66 #define DEFAULT_GO_HISPEED_LOAD 85
  67 static unsigned long go_hispeed_load;
  68
  69 /* Target load.  Lower values result in higher CPU speeds. */
  70 #define DEFAULT_TARGET_LOAD 90
  71 static unsigned int default_target_loads[] = {DEFAULT_TARGET_LOAD};
  72 static spinlock_t target_loads_lock;
  73 static unsigned int *target_loads = default_target_loads;
  74 static int ntarget_loads = ARRAY_SIZE(default_target_loads);
  75
  76 /*
  77  * The minimum amount of time to spend at a frequency before we can ramp down.
  78  */
  79 #define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
  80 static unsigned long min_sample_time;
  81
  82 /*
  83  * The sample rate of the timer used to increase frequency
  84  */
  85 #define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC)
  86 static unsigned long timer_rate;
  87
  88 /*
  89  * Wait this long before raising speed above hispeed, by default a single
  90  * timer interval.
  91  */
  92 #define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE
  93 static unsigned long above_hispeed_delay_val;
  94
  95 /*
  96  * Non-zero means longer-term speed boost active.
  97  */
  98
  99 static int boost_val;
 100
 101 static bool governidle;
 102 module_param(governidle, bool, S_IWUSR | S_IRUGO);
 103 MODULE_PARM_DESC(governidle,
 104         "Set to 1 to wake up CPUs from idle to reduce speed (default 0)");
 105
 106 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 107                 unsigned int event);
 108
 109 #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 110 static
 111 #endif
 112 struct cpufreq_governor cpufreq_gov_interactive = {
 113         .name = "interactive",
 114         .governor = cpufreq_governor_interactive,
 115         .max_transition_latency = 10000000,
 116         .owner = THIS_MODULE,
 117 };
 118
 119 static void cpufreq_interactive_timer_resched(
 120         struct cpufreq_interactive_cpuinfo *pcpu)
 121 {
 122         mod_timer_pinned(&pcpu->cpu_timer,
 123                          jiffies + usecs_to_jiffies(timer_rate));
 124         pcpu->time_in_idle =
 125                 get_cpu_idle_time_us(smp_processor_id(),
 126                                      &pcpu->time_in_idle_timestamp);
 127 }
 128
 129 static unsigned int freq_to_targetload(unsigned int freq)
 130 {
 131         int i;
 132         unsigned int ret;
 133
 134         spin_lock(&target_loads_lock);
 135
 136         for (i = 0; i < ntarget_loads - 1 && freq >= target_loads[i+1]; i += 2)
 137                 ;
 138
 139         ret = target_loads[i];
 140         spin_unlock(&target_loads_lock);
 141         return ret;
 142 }
 143
 144 /*
 145  * If increasing frequencies never map to a lower target load then
 146  * choose_freq() will find the minimum frequency that does not exceed its
 147  * target load given the current load.
 148  */
 149
 150 static unsigned int choose_freq(
 151         struct cpufreq_interactive_cpuinfo *pcpu, unsigned int curload)
 152 {
 153         unsigned int freq = pcpu->policy->cur;
 154         unsigned int loadadjfreq = freq * curload;
 155         unsigned int prevfreq, freqmin, freqmax;
 156         unsigned int tl;
 157         int index;
 158
 159         freqmin = 0;
 160         freqmax = UINT_MAX;
 161
 162         do {
 163                 prevfreq = freq;
 164                 tl = freq_to_targetload(freq);
 165
 166                 /*
 167                  * Find the lowest frequency where the computed load is less
 168                  * than or equal to the target load.
 169                  */
 170
 171                 cpufreq_frequency_table_target(
 172                         pcpu->policy, pcpu->freq_table, loadadjfreq / tl,
 173                         CPUFREQ_RELATION_L, &index);
 174                 freq = pcpu->freq_table[index].frequency;
 175
 176                 if (freq > prevfreq) {
 177                         /* The previous frequency is too low. */
 178                         freqmin = prevfreq;
 179
 180                         if (freq >= freqmax) {
 181                                 /*
 182                                  * Find the highest frequency that is less
 183                                  * than freqmax.
 184                                  */
 185                                 cpufreq_frequency_table_target(
 186                                         pcpu->policy, pcpu->freq_table,
 187                                         freqmax - 1, CPUFREQ_RELATION_H,
 188                                         &index);
 189                                 freq = pcpu->freq_table[index].frequency;
 190
 191                                 if (freq == freqmin) {
 192                                         /*
 193                                          * The first frequency below freqmax
 194                                          * has already been found to be too
 195                                          * low.  freqmax is the lowest speed
 196                                          * we found that is fast enough.
 197                                          */
 198                                         freq = freqmax;
 199                                         break;
 200                                 }
 201                         }
 202                 } else if (freq < prevfreq) {
 203                         /* The previous frequency is high enough. */
 204                         freqmax = prevfreq;
 205
 206                         if (freq <= freqmin) {
 207                                 /*
 208                                  * Find the lowest frequency that is higher
 209                                  * than freqmin.
 210                                  */
 211                                 cpufreq_frequency_table_target(
 212                                         pcpu->policy, pcpu->freq_table,
 213                                         freqmin + 1, CPUFREQ_RELATION_L,
 214                                         &index);
 215                                 freq = pcpu->freq_table[index].frequency;
 216
 217                                 /*
 218                                  * If freqmax is the first frequency above
 219                                  * freqmin then we have already found that
 220                                  * this speed is fast enough.
 221                                  */
 222                                 if (freq == freqmax)
 223                                         break;
 224                         }
 225                 }
 226
 227                 /* If same frequency chosen as previous then done. */
 228         } while (freq != prevfreq);
 229
 230         return freq;
 231 }
 232
 233 static void cpufreq_interactive_timer(unsigned long data)
 234 {
 235         u64 now;
 236         unsigned int delta_idle;
 237         unsigned int delta_time;
 238         int cpu_load;
 239         struct cpufreq_interactive_cpuinfo *pcpu =
 240                 &per_cpu(cpuinfo, data);
 241         u64 now_idle;
 242         unsigned int new_freq;
 243         unsigned int index;
 244         unsigned long flags;
 245
 246         smp_rmb();
 247
 248         if (!pcpu->governor_enabled)
 249                 goto exit;
 250
 251         now_idle = get_cpu_idle_time_us(data, &now);
 252         delta_idle = (unsigned int)(now_idle - pcpu->time_in_idle);
 253         delta_time = (unsigned int)(now - pcpu->time_in_idle_timestamp);
 254
 255         /*
 256          * If timer ran less than 1ms after short-term sample started, retry.
 257          */
 258         if (delta_time < 1000)
 259                 goto rearm;
 260
 261         if (delta_idle > delta_time)
 262                 cpu_load = 0;
 263         else
 264                 cpu_load = 100 * (delta_time - delta_idle) / delta_time;
 265
 266         if ((cpu_load >= go_hispeed_load || boost_val) &&
 267             pcpu->target_freq < hispeed_freq)
 268                 new_freq = hispeed_freq;
 269         else
 270                 new_freq = choose_freq(pcpu, cpu_load);
 271
 272         if (pcpu->target_freq >= hispeed_freq &&
 273             new_freq > pcpu->target_freq &&
 274             now - pcpu->hispeed_validate_time < above_hispeed_delay_val) {
 275                 trace_cpufreq_interactive_notyet(
 276                         data, cpu_load, pcpu->target_freq,
 277                         pcpu->policy->cur, new_freq);
 278                 goto rearm;
 279         }
 280
 281         pcpu->hispeed_validate_time = now;
 282
 283         if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
 284                                            new_freq, CPUFREQ_RELATION_L,
 285                                            &index)) {
 286                 pr_warn_once("timer %d: cpufreq_frequency_table_target error\n",
 287                              (int) data);
 288                 goto rearm;
 289         }
 290
 291         new_freq = pcpu->freq_table[index].frequency;
 292
 293         /*
 294          * Do not scale below floor_freq unless we have been at or above the
 295          * floor frequency for the minimum sample time since last validated.
 296          */
 297         if (new_freq < pcpu->floor_freq) {
 298                 if (now - pcpu->floor_validate_time < min_sample_time) {
 299                         trace_cpufreq_interactive_notyet(
 300                                 data, cpu_load, pcpu->target_freq,
 301                                 pcpu->policy->cur, new_freq);
 302                         goto rearm;
 303                 }
 304         }
 305
 306         pcpu->floor_freq = new_freq;
 307         pcpu->floor_validate_time = now;
 308
 309         if (pcpu->target_freq == new_freq) {
 310                 trace_cpufreq_interactive_already(
 311                         data, cpu_load, pcpu->target_freq,
 312                         pcpu->policy->cur, new_freq);
 313                 goto rearm_if_notmax;
 314         }
 315
 316         trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
 317                                          pcpu->policy->cur, new_freq);
 318
 319         pcpu->target_freq = new_freq;
 320         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 321         cpumask_set_cpu(data, &speedchange_cpumask);
 322         spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 323         wake_up_process(speedchange_task);
 324
 325 rearm_if_notmax:
 326         /*
 327          * Already set max speed and don't see a need to change that,
 328          * wait until next idle to re-evaluate, don't need timer.
 329          */
 330         if (pcpu->target_freq == pcpu->policy->max)
 331                 goto exit;
 332
 333 rearm:
 334         if (!timer_pending(&pcpu->cpu_timer)) {
 335                 /*
 336                  * If governing speed in idle and already at min, cancel the
 337                  * timer if that CPU goes idle.  We don't need to re-evaluate
 338                  * speed until the next idle exit.
 339                  */
 340                 if (governidle && pcpu->target_freq == pcpu->policy->min)
 341                         pcpu->timer_idlecancel = 1;
 342
 343                 cpufreq_interactive_timer_resched(pcpu);
 344         }
 345
 346 exit:
 347         return;
 348 }
 349
 350 static void cpufreq_interactive_idle_start(void)
 351 {
 352         struct cpufreq_interactive_cpuinfo *pcpu =
 353                 &per_cpu(cpuinfo, smp_processor_id());
 354         int pending;
 355
 356         if (!pcpu->governor_enabled)
 357                 return;
 358
 359         pending = timer_pending(&pcpu->cpu_timer);
 360
 361         if (pcpu->target_freq != pcpu->policy->min) {
 362                 /*
 363                  * Entering idle while not at lowest speed.  On some
 364                  * platforms this can hold the other CPU(s) at that speed
 365                  * even though the CPU is idle. Set a timer to re-evaluate
 366                  * speed so this idle CPU doesn't hold the other CPUs above
 367                  * min indefinitely.  This should probably be a quirk of
 368                  * the CPUFreq driver.
 369                  */
 370                 if (!pending) {
 371                         pcpu->timer_idlecancel = 0;
 372                         cpufreq_interactive_timer_resched(pcpu);
 373                 }
 374         } else if (governidle) {
 375                 /*
 376                  * If at min speed and entering idle after load has
 377                  * already been evaluated, and a timer has been set just in
 378                  * case the CPU suddenly goes busy, cancel that timer.  The
 379                  * CPU didn't go busy; we'll recheck things upon idle exit.
 380                  */
 381                 if (pending && pcpu->timer_idlecancel) {
 382                         del_timer(&pcpu->cpu_timer);
 383                         pcpu->timer_idlecancel = 0;
 384                 }
 385         }
 386
 387 }
 388
 389 static void cpufreq_interactive_idle_end(void)
 390 {
 391         struct cpufreq_interactive_cpuinfo *pcpu =
 392                 &per_cpu(cpuinfo, smp_processor_id());
 393
 394         if (!pcpu->governor_enabled)
 395                 return;
 396
 397         /* Arm the timer for 1-2 ticks later if not already. */
 398         if (!timer_pending(&pcpu->cpu_timer)) {
 399                 pcpu->timer_idlecancel = 0;
 400                 cpufreq_interactive_timer_resched(pcpu);
 401         } else if (!governidle &&
 402                    time_after_eq(jiffies, pcpu->cpu_timer.expires)) {
 403                 del_timer(&pcpu->cpu_timer);
 404                 cpufreq_interactive_timer(smp_processor_id());
 405         }
 406 }
 407
 408 static int cpufreq_interactive_speedchange_task(void *data)
 409 {
 410         unsigned int cpu;
 411         cpumask_t tmp_mask;
 412         unsigned long flags;
 413         struct cpufreq_interactive_cpuinfo *pcpu;
 414
 415         while (1) {
 416                 set_current_state(TASK_INTERRUPTIBLE);
 417                 spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 418
 419                 if (cpumask_empty(&speedchange_cpumask)) {
 420                         spin_unlock_irqrestore(&speedchange_cpumask_lock,
 421                                                flags);
 422                         schedule();
 423
 424                         if (kthread_should_stop())
 425                                 break;
 426
 427                         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 428                 }
 429
 430                 set_current_state(TASK_RUNNING);
 431                 tmp_mask = speedchange_cpumask;
 432                 cpumask_clear(&speedchange_cpumask);
 433                 spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 434
 435                 for_each_cpu(cpu, &tmp_mask) {
 436                         unsigned int j;
 437                         unsigned int max_freq = 0;
 438
 439                         pcpu = &per_cpu(cpuinfo, cpu);
 440                         smp_rmb();
 441
 442                         if (!pcpu->governor_enabled)
 443                                 continue;
 444
 445                         for_each_cpu(j, pcpu->policy->cpus) {
 446                                 struct cpufreq_interactive_cpuinfo *pjcpu =
 447                                         &per_cpu(cpuinfo, j);
 448
 449                                 if (pjcpu->target_freq > max_freq)
 450                                         max_freq = pjcpu->target_freq;
 451                         }
 452
 453                         if (max_freq != pcpu->policy->cur)
 454                                 __cpufreq_driver_target(pcpu->policy,
 455                                                         max_freq,
 456                                                         CPUFREQ_RELATION_H);
 457                         trace_cpufreq_interactive_setspeed(cpu,
 458                                                      pcpu->target_freq,
 459                                                      pcpu->policy->cur);
 460                 }
 461         }
 462
 463         return 0;
 464 }
 465
 466 static void cpufreq_interactive_boost(void)
 467 {
 468         int i;
 469         int anyboost = 0;
 470         unsigned long flags;
 471         struct cpufreq_interactive_cpuinfo *pcpu;
 472
 473         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 474
 475         for_each_online_cpu(i) {
 476                 pcpu = &per_cpu(cpuinfo, i);
 477
 478                 if (pcpu->target_freq < hispeed_freq) {
 479                         pcpu->target_freq = hispeed_freq;
 480                         cpumask_set_cpu(i, &speedchange_cpumask);
 481                         pcpu->hispeed_validate_time =
 482                                 ktime_to_us(ktime_get());
 483                         anyboost = 1;
 484                 }
 485
 486                 /*
 487                  * Set floor freq and (re)start timer for when last
 488                  * validated.
 489                  */
 490
 491                 pcpu->floor_freq = hispeed_freq;
 492                 pcpu->floor_validate_time = ktime_to_us(ktime_get());
 493         }
 494
 495         spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 496
 497         if (anyboost)
 498                 wake_up_process(speedchange_task);
 499 }
 500
 501 static ssize_t show_target_loads(
 502         struct kobject *kobj, struct attribute *attr, char *buf)
 503 {
 504         int i;
 505         ssize_t ret = 0;
 506
 507         spin_lock(&target_loads_lock);
 508
 509         for (i = 0; i < ntarget_loads; i++)
 510                 ret += sprintf(buf + ret, "%u%s", target_loads[i],
 511                                i & 0x1 ? ":" : " ");
 512
 513         ret += sprintf(buf + ret, "\n");
 514         spin_unlock(&target_loads_lock);
 515         return ret;
 516 }
 517
 518 static ssize_t store_target_loads(
 519         struct kobject *kobj, struct attribute *attr, const char *buf,
 520         size_t count)
 521 {
 522         int ret;
 523         const char *cp;
 524         unsigned int *new_target_loads = NULL;
 525         int ntokens = 1;
 526         int i;
 527
 528         cp = buf;
 529         while ((cp = strpbrk(cp + 1, " :")))
 530                 ntokens++;
 531
 532         if (!(ntokens & 0x1))
 533                 goto err_inval;
 534
 535         new_target_loads = kmalloc(ntokens * sizeof(unsigned int), GFP_KERNEL);
 536         if (!new_target_loads) {
 537                 ret = -ENOMEM;
 538                 goto err;
 539         }
 540
 541         cp = buf;
 542         i = 0;
 543         while (i < ntokens) {
 544                 if (sscanf(cp, "%u", &new_target_loads[i++]) != 1)
 545                         goto err_inval;
 546
 547                 cp = strpbrk(cp, " :");
 548                 if (!cp)
 549                         break;
 550                 cp++;
 551         }
 552
 553         if (i != ntokens)
 554                 goto err_inval;
 555
 556         spin_lock(&target_loads_lock);
 557         if (target_loads != default_target_loads)
 558                 kfree(target_loads);
 559         target_loads = new_target_loads;
 560         ntarget_loads = ntokens;
 561         spin_unlock(&target_loads_lock);
 562         return count;
 563
 564 err_inval:
 565         ret = -EINVAL;
 566 err:
 567         kfree(new_target_loads);
 568         return ret;
 569 }
 570
 571 static struct global_attr target_loads_attr =
 572         __ATTR(target_loads, S_IRUGO | S_IWUSR,
 573                 show_target_loads, store_target_loads);
 574
 575 static ssize_t show_hispeed_freq(struct kobject *kobj,
 576                                  struct attribute *attr, char *buf)
 577 {
 578         return sprintf(buf, "%u\n", hispeed_freq);
 579 }
 580
 581 static ssize_t store_hispeed_freq(struct kobject *kobj,
 582                                   struct attribute *attr, const char *buf,
 583                                   size_t count)
 584 {
 585         int ret;
 586         long unsigned int val;
 587
 588         ret = strict_strtoul(buf, 0, &val);
 589         if (ret < 0)
 590                 return ret;
 591         hispeed_freq = val;
 592         return count;
 593 }
 594
 595 static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644,
 596                 show_hispeed_freq, store_hispeed_freq);
 597
 598
 599 static ssize_t show_go_hispeed_load(struct kobject *kobj,
 600                                      struct attribute *attr, char *buf)
 601 {
 602         return sprintf(buf, "%lu\n", go_hispeed_load);
 603 }
 604
 605 static ssize_t store_go_hispeed_load(struct kobject *kobj,
 606                         struct attribute *attr, const char *buf, size_t count)
 607 {
 608         int ret;
 609         unsigned long val;
 610
 611         ret = strict_strtoul(buf, 0, &val);
 612         if (ret < 0)
 613                 return ret;
 614         go_hispeed_load = val;
 615         return count;
 616 }
 617
 618 static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644,
 619                 show_go_hispeed_load, store_go_hispeed_load);
 620
 621 static ssize_t show_min_sample_time(struct kobject *kobj,
 622                                 struct attribute *attr, char *buf)
 623 {
 624         return sprintf(buf, "%lu\n", min_sample_time);
 625 }
 626
 627 static ssize_t store_min_sample_time(struct kobject *kobj,
 628                         struct attribute *attr, const char *buf, size_t count)
 629 {
 630         int ret;
 631         unsigned long val;
 632
 633         ret = strict_strtoul(buf, 0, &val);
 634         if (ret < 0)
 635                 return ret;
 636         min_sample_time = val;
 637         return count;
 638 }
 639
 640 static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
 641                 show_min_sample_time, store_min_sample_time);
 642
 643 static ssize_t show_above_hispeed_delay(struct kobject *kobj,
 644                                         struct attribute *attr, char *buf)
 645 {
 646         return sprintf(buf, "%lu\n", above_hispeed_delay_val);
 647 }
 648
 649 static ssize_t store_above_hispeed_delay(struct kobject *kobj,
 650                                          struct attribute *attr,
 651                                          const char *buf, size_t count)
 652 {
 653         int ret;
 654         unsigned long val;
 655
 656         ret = strict_strtoul(buf, 0, &val);
 657         if (ret < 0)
 658                 return ret;
 659         above_hispeed_delay_val = val;
 660         return count;
 661 }
 662
 663 define_one_global_rw(above_hispeed_delay);
 664
 665 static ssize_t show_timer_rate(struct kobject *kobj,
 666                         struct attribute *attr, char *buf)
 667 {
 668         return sprintf(buf, "%lu\n", timer_rate);
 669 }
 670
 671 static ssize_t store_timer_rate(struct kobject *kobj,
 672                         struct attribute *attr, const char *buf, size_t count)
 673 {
 674         int ret;
 675         unsigned long val;
 676
 677         ret = strict_strtoul(buf, 0, &val);
 678         if (ret < 0)
 679                 return ret;
 680         timer_rate = val;
 681         return count;
 682 }
 683
 684 static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
 685                 show_timer_rate, store_timer_rate);
 686
 687 static ssize_t show_boost(struct kobject *kobj, struct attribute *attr,
 688                           char *buf)
 689 {
 690         return sprintf(buf, "%d\n", boost_val);
 691 }
 692
 693 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 694                            const char *buf, size_t count)
 695 {
 696         int ret;
 697         unsigned long val;
 698
 699         ret = kstrtoul(buf, 0, &val);
 700         if (ret < 0)
 701                 return ret;
 702
 703         boost_val = val;
 704
 705         if (boost_val) {
 706                 trace_cpufreq_interactive_boost("on");
 707                 cpufreq_interactive_boost();
 708         } else {
 709                 trace_cpufreq_interactive_unboost("off");
 710         }
 711
 712         return count;
 713 }
 714
 715 define_one_global_rw(boost);
 716
 717 static ssize_t store_boostpulse(struct kobject *kobj, struct attribute *attr,
 718                                 const char *buf, size_t count)
 719 {
 720         int ret;
 721         unsigned long val;
 722
 723         ret = kstrtoul(buf, 0, &val);
 724         if (ret < 0)
 725                 return ret;
 726
 727         trace_cpufreq_interactive_boost("pulse");
 728         cpufreq_interactive_boost();
 729         return count;
 730 }
 731
 732 static struct global_attr boostpulse =
 733         __ATTR(boostpulse, 0200, NULL, store_boostpulse);
 734
 735 static struct attribute *interactive_attributes[] = {
 736         &target_loads_attr.attr,
 737         &hispeed_freq_attr.attr,
 738         &go_hispeed_load_attr.attr,
 739         &above_hispeed_delay.attr,
 740         &min_sample_time_attr.attr,
 741         &timer_rate_attr.attr,
 742         &boost.attr,
 743         &boostpulse.attr,
 744         NULL,
 745 };
 746
 747 static struct attribute_group interactive_attr_group = {
 748         .attrs = interactive_attributes,
 749         .name = "interactive",
 750 };
 751
 752 static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
 753                                              unsigned long val,
 754                                              void *data)
 755 {
 756         switch (val) {
 757         case IDLE_START:
 758                 cpufreq_interactive_idle_start();
 759                 break;
 760         case IDLE_END:
 761                 cpufreq_interactive_idle_end();
 762                 break;
 763         }
 764
 765         return 0;
 766 }
 767
 768 static struct notifier_block cpufreq_interactive_idle_nb = {
 769         .notifier_call = cpufreq_interactive_idle_notifier,
 770 };
 771
 772 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 773                 unsigned int event)
 774 {
 775         int rc;
 776         unsigned int j;
 777         struct cpufreq_interactive_cpuinfo *pcpu;
 778         struct cpufreq_frequency_table *freq_table;
 779
 780         switch (event) {
 781         case CPUFREQ_GOV_START:
 782                 if (!cpu_online(policy->cpu))
 783                         return -EINVAL;
 784
 785                 freq_table =
 786                         cpufreq_frequency_get_table(policy->cpu);
 787                 if (!hispeed_freq)
 788                         hispeed_freq = policy->max;
 789
 790                 for_each_cpu(j, policy->cpus) {
 791                         pcpu = &per_cpu(cpuinfo, j);
 792                         pcpu->policy = policy;
 793                         pcpu->target_freq = policy->cur;
 794                         pcpu->freq_table = freq_table;
 795                         pcpu->floor_freq = pcpu->target_freq;
 796                         pcpu->floor_validate_time =
 797                                 ktime_to_us(ktime_get());
 798                         pcpu->hispeed_validate_time =
 799                                 pcpu->floor_validate_time;
 800                         pcpu->governor_enabled = 1;
 801                         smp_wmb();
 802                         pcpu->cpu_timer.expires =
 803                                 jiffies + usecs_to_jiffies(timer_rate);
 804                         add_timer_on(&pcpu->cpu_timer, j);
 805                 }
 806
 807                 /*
 808                  * Do not register the idle hook and create sysfs
 809                  * entries if we have already done so.
 810                  */
 811                 if (atomic_inc_return(&active_count) > 1)
 812                         return 0;
 813
 814                 rc = sysfs_create_group(cpufreq_global_kobject,
 815                                 &interactive_attr_group);
 816                 if (rc)
 817                         return rc;
 818
 819                 idle_notifier_register(&cpufreq_interactive_idle_nb);
 820                 break;
 821
 822         case CPUFREQ_GOV_STOP:
 823                 for_each_cpu(j, policy->cpus) {
 824                         pcpu = &per_cpu(cpuinfo, j);
 825                         pcpu->governor_enabled = 0;
 826                         smp_wmb();
 827                         del_timer_sync(&pcpu->cpu_timer);
 828                 }
 829
 830                 if (atomic_dec_return(&active_count) > 0)
 831                         return 0;
 832
 833                 idle_notifier_unregister(&cpufreq_interactive_idle_nb);
 834                 sysfs_remove_group(cpufreq_global_kobject,
 835                                 &interactive_attr_group);
 836
 837                 break;
 838
 839         case CPUFREQ_GOV_LIMITS:
 840                 if (policy->max < policy->cur)
 841                         __cpufreq_driver_target(policy,
 842                                         policy->max, CPUFREQ_RELATION_H);
 843                 else if (policy->min > policy->cur)
 844                         __cpufreq_driver_target(policy,
 845                                         policy->min, CPUFREQ_RELATION_L);
 846                 break;
 847         }
 848         return 0;
 849 }
 850
 851 static int __init cpufreq_interactive_init(void)
 852 {
 853         unsigned int i;
 854         struct cpufreq_interactive_cpuinfo *pcpu;
 855         struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 856
 857         go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
 858         min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
 859         above_hispeed_delay_val = DEFAULT_ABOVE_HISPEED_DELAY;
 860         timer_rate = DEFAULT_TIMER_RATE;
 861
 862         /* Initalize per-cpu timers */
 863         for_each_possible_cpu(i) {
 864                 pcpu = &per_cpu(cpuinfo, i);
 865                 if (governidle)
 866                         init_timer(&pcpu->cpu_timer);
 867                 else
 868                         init_timer_deferrable(&pcpu->cpu_timer);
 869                 pcpu->cpu_timer.function = cpufreq_interactive_timer;
 870                 pcpu->cpu_timer.data = i;
 871         }
 872
 873         spin_lock_init(&target_loads_lock);
 874         spin_lock_init(&speedchange_cpumask_lock);
 875         speedchange_task =
 876                 kthread_create(cpufreq_interactive_speedchange_task, NULL,
 877                                "cfinteractive");
 878         if (IS_ERR(speedchange_task))
 879                 return PTR_ERR(speedchange_task);
 880
 881         sched_setscheduler_nocheck(speedchange_task, SCHED_FIFO, &param);
 882         get_task_struct(speedchange_task);
 883
 884         /* NB: wake up so the thread does not look hung to the freezer */
 885         wake_up_process(speedchange_task);
 886
 887         return cpufreq_register_governor(&cpufreq_gov_interactive);
 888 }
 889
 890 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 891 fs_initcall(cpufreq_interactive_init);
 892 #else
 893 module_init(cpufreq_interactive_init);
 894 #endif
 895
 896 static void __exit cpufreq_interactive_exit(void)
 897 {
 898         cpufreq_unregister_governor(&cpufreq_gov_interactive);
 899         kthread_stop(speedchange_task);
 900         put_task_struct(speedchange_task);
 901 }
 902
 903 module_exit(cpufreq_interactive_exit);
 904
 905 MODULE_AUTHOR("Mike Chan <mike@android.com>");
 906 MODULE_DESCRIPTION("'cpufreq_interactive' - A cpufreq governor for "
 907         "Latency sensitive workloads");
 908 MODULE_LICENSE("GPL");