drivers/cpufreq/cpufreq_interactive.c

   1 /*
   2  * drivers/cpufreq/cpufreq_interactive.c
   3  *
   4  * Copyright (C) 2010 Google, Inc.
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * Author: Mike Chan (mike@android.com)
  16  *
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/cpumask.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/module.h>
  23 #include <linux/moduleparam.h>
  24 #include <linux/rwsem.h>
  25 #include <linux/sched.h>
  26 #include <linux/sched/rt.h>
  27 #include <linux/tick.h>
  28 #include <linux/time.h>
  29 #include <linux/timer.h>
  30 #include <linux/workqueue.h>
  31 #include <linux/kthread.h>
  32 #include <linux/slab.h>
  33 #include <linux/kernel_stat.h>
  34 #include <asm/cputime.h>
  35
  36 #define CREATE_TRACE_POINTS
  37 #include <trace/events/cpufreq_interactive.h>
  38
  39 static int active_count;
  40
  41 struct cpufreq_interactive_cpuinfo {
  42         struct timer_list cpu_timer;
  43         struct timer_list cpu_slack_timer;
  44         spinlock_t load_lock; /* protects the next 4 fields */
  45         u64 time_in_idle;
  46         u64 time_in_idle_timestamp;
  47         u64 cputime_speedadj;
  48         u64 cputime_speedadj_timestamp;
  49         struct cpufreq_policy *policy;
  50         struct cpufreq_frequency_table *freq_table;
  51         unsigned int target_freq;
  52         unsigned int floor_freq;
  53         u64 floor_validate_time;
  54         u64 hispeed_validate_time;
  55         struct rw_semaphore enable_sem;
  56         int governor_enabled;
  57 };
  58
  59 static DEFINE_PER_CPU(struct cpufreq_interactive_cpuinfo, cpuinfo);
  60
  61 /* realtime thread handles frequency scaling */
  62 static struct task_struct *speedchange_task;
  63 static cpumask_t speedchange_cpumask;
  64 static spinlock_t speedchange_cpumask_lock;
  65 static struct mutex gov_lock;
  66
  67 /* Hi speed to bump to from lo speed when load burst (default max) */
  68 static unsigned int hispeed_freq;
  69
  70 /* Go to hi speed when CPU load at or above this value. */
  71 #define DEFAULT_GO_HISPEED_LOAD 99
  72 static unsigned long go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
  73
  74 /* Target load.  Lower values result in higher CPU speeds. */
  75 #define DEFAULT_TARGET_LOAD 90
  76 static unsigned int default_target_loads[] = {DEFAULT_TARGET_LOAD};
  77 static spinlock_t target_loads_lock;
  78 static unsigned int *target_loads = default_target_loads;
  79 static int ntarget_loads = ARRAY_SIZE(default_target_loads);
  80
  81 /*
  82  * The minimum amount of time to spend at a frequency before we can ramp down.
  83  */
  84 #define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
  85 static unsigned long min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
  86
  87 /*
  88  * The sample rate of the timer used to increase frequency
  89  */
  90 #define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC)
  91 static unsigned long timer_rate = DEFAULT_TIMER_RATE;
  92
  93 /*
  94  * Wait this long before raising speed above hispeed, by default a single
  95  * timer interval.
  96  */
  97 #define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE
  98 static unsigned int default_above_hispeed_delay[] = {
  99         DEFAULT_ABOVE_HISPEED_DELAY };
 100 static spinlock_t above_hispeed_delay_lock;
 101 static unsigned int *above_hispeed_delay = default_above_hispeed_delay;
 102 static int nabove_hispeed_delay = ARRAY_SIZE(default_above_hispeed_delay);
 103
 104 /* Non-zero means indefinite speed boost active */
 105 static int boost_val;
 106 /* Duration of a boot pulse in usecs */
 107 static int boostpulse_duration_val = DEFAULT_MIN_SAMPLE_TIME;
 108 /* End time of boost pulse in ktime converted to usecs */
 109 static u64 boostpulse_endtime;
 110
 111 /*
 112  * Max additional time to wait in idle, beyond timer_rate, at speeds above
 113  * minimum before wakeup to reduce speed, or -1 if unnecessary.
 114  */
 115 #define DEFAULT_TIMER_SLACK (4 * DEFAULT_TIMER_RATE)
 116 static int timer_slack_val = DEFAULT_TIMER_SLACK;
 117
 118 static bool io_is_busy;
 119
 120 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 121                 unsigned int event);
 122
 123 #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 124 static
 125 #endif
 126 struct cpufreq_governor cpufreq_gov_interactive = {
 127         .name = "interactive",
 128         .governor = cpufreq_governor_interactive,
 129         .max_transition_latency = 10000000,
 130         .owner = THIS_MODULE,
 131 };
 132
 133 static inline cputime64_t get_cpu_idle_time_jiffy(unsigned int cpu,
 134                                                   cputime64_t *wall)
 135 {
 136         u64 idle_time;
 137         u64 cur_wall_time;
 138         u64 busy_time;
 139
 140         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 141
 142         busy_time  = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 143         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 144         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 145         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 146         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 147         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 148
 149         idle_time = cur_wall_time - busy_time;
 150         if (wall)
 151                 *wall = jiffies_to_usecs(cur_wall_time);
 152
 153         return jiffies_to_usecs(idle_time);
 154 }
 155
 156 static inline cputime64_t get_cpu_idle_time(unsigned int cpu,
 157                                             cputime64_t *wall)
 158 {
 159         u64 idle_time = get_cpu_idle_time_us(cpu, wall);
 160
 161         if (idle_time == -1ULL)
 162                 idle_time = get_cpu_idle_time_jiffy(cpu, wall);
 163         else if (!io_is_busy)
 164                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 165
 166         return idle_time;
 167 }
 168
 169 static void cpufreq_interactive_timer_resched(
 170         struct cpufreq_interactive_cpuinfo *pcpu)
 171 {
 172         unsigned long expires = jiffies + usecs_to_jiffies(timer_rate);
 173         unsigned long flags;
 174
 175         mod_timer_pinned(&pcpu->cpu_timer, expires);
 176         if (timer_slack_val >= 0 && pcpu->target_freq > pcpu->policy->min) {
 177                 expires += usecs_to_jiffies(timer_slack_val);
 178                 mod_timer_pinned(&pcpu->cpu_slack_timer, expires);
 179         }
 180
 181         spin_lock_irqsave(&pcpu->load_lock, flags);
 182         pcpu->time_in_idle =
 183                 get_cpu_idle_time(smp_processor_id(),
 184                                      &pcpu->time_in_idle_timestamp);
 185         pcpu->cputime_speedadj = 0;
 186         pcpu->cputime_speedadj_timestamp = pcpu->time_in_idle_timestamp;
 187         spin_unlock_irqrestore(&pcpu->load_lock, flags);
 188 }
 189
 190 static unsigned int freq_to_above_hispeed_delay(unsigned int freq)
 191 {
 192         int i;
 193         unsigned int ret;
 194         unsigned long flags;
 195
 196         spin_lock_irqsave(&above_hispeed_delay_lock, flags);
 197
 198         for (i = 0; i < nabove_hispeed_delay - 1 &&
 199                         freq >= above_hispeed_delay[i+1]; i += 2)
 200                 ;
 201
 202         ret = above_hispeed_delay[i];
 203         spin_unlock_irqrestore(&above_hispeed_delay_lock, flags);
 204         return ret;
 205 }
 206
 207 static unsigned int freq_to_targetload(unsigned int freq)
 208 {
 209         int i;
 210         unsigned int ret;
 211         unsigned long flags;
 212
 213         spin_lock_irqsave(&target_loads_lock, flags);
 214
 215         for (i = 0; i < ntarget_loads - 1 && freq >= target_loads[i+1]; i += 2)
 216                 ;
 217
 218         ret = target_loads[i];
 219         spin_unlock_irqrestore(&target_loads_lock, flags);
 220         return ret;
 221 }
 222
 223 /*
 224  * If increasing frequencies never map to a lower target load then
 225  * choose_freq() will find the minimum frequency that does not exceed its
 226  * target load given the current load.
 227  */
 228
 229 static unsigned int choose_freq(
 230         struct cpufreq_interactive_cpuinfo *pcpu, unsigned int loadadjfreq)
 231 {
 232         unsigned int freq = pcpu->policy->cur;
 233         unsigned int prevfreq, freqmin, freqmax;
 234         unsigned int tl;
 235         int index;
 236
 237         freqmin = 0;
 238         freqmax = UINT_MAX;
 239
 240         do {
 241                 prevfreq = freq;
 242                 tl = freq_to_targetload(freq);
 243
 244                 /*
 245                  * Find the lowest frequency where the computed load is less
 246                  * than or equal to the target load.
 247                  */
 248
 249                 cpufreq_frequency_table_target(
 250                         pcpu->policy, pcpu->freq_table, loadadjfreq / tl,
 251                         CPUFREQ_RELATION_L, &index);
 252                 freq = pcpu->freq_table[index].frequency;
 253
 254                 if (freq > prevfreq) {
 255                         /* The previous frequency is too low. */
 256                         freqmin = prevfreq;
 257
 258                         if (freq >= freqmax) {
 259                                 /*
 260                                  * Find the highest frequency that is less
 261                                  * than freqmax.
 262                                  */
 263                                 cpufreq_frequency_table_target(
 264                                         pcpu->policy, pcpu->freq_table,
 265                                         freqmax - 1, CPUFREQ_RELATION_H,
 266                                         &index);
 267                                 freq = pcpu->freq_table[index].frequency;
 268
 269                                 if (freq == freqmin) {
 270                                         /*
 271                                          * The first frequency below freqmax
 272                                          * has already been found to be too
 273                                          * low.  freqmax is the lowest speed
 274                                          * we found that is fast enough.
 275                                          */
 276                                         freq = freqmax;
 277                                         break;
 278                                 }
 279                         }
 280                 } else if (freq < prevfreq) {
 281                         /* The previous frequency is high enough. */
 282                         freqmax = prevfreq;
 283
 284                         if (freq <= freqmin) {
 285                                 /*
 286                                  * Find the lowest frequency that is higher
 287                                  * than freqmin.
 288                                  */
 289                                 cpufreq_frequency_table_target(
 290                                         pcpu->policy, pcpu->freq_table,
 291                                         freqmin + 1, CPUFREQ_RELATION_L,
 292                                         &index);
 293                                 freq = pcpu->freq_table[index].frequency;
 294
 295                                 /*
 296                                  * If freqmax is the first frequency above
 297                                  * freqmin then we have already found that
 298                                  * this speed is fast enough.
 299                                  */
 300                                 if (freq == freqmax)
 301                                         break;
 302                         }
 303                 }
 304
 305                 /* If same frequency chosen as previous then done. */
 306         } while (freq != prevfreq);
 307
 308         return freq;
 309 }
 310
 311 static u64 update_load(int cpu)
 312 {
 313         struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, cpu);
 314         u64 now;
 315         u64 now_idle;
 316         unsigned int delta_idle;
 317         unsigned int delta_time;
 318         u64 active_time;
 319
 320         now_idle = get_cpu_idle_time(cpu, &now);
 321         delta_idle = (unsigned int)(now_idle - pcpu->time_in_idle);
 322         delta_time = (unsigned int)(now - pcpu->time_in_idle_timestamp);
 323         active_time = delta_time - delta_idle;
 324         pcpu->cputime_speedadj += active_time * pcpu->policy->cur;
 325
 326         pcpu->time_in_idle = now_idle;
 327         pcpu->time_in_idle_timestamp = now;
 328         return now;
 329 }
 330
 331 static void cpufreq_interactive_timer(unsigned long data)
 332 {
 333         u64 now;
 334         unsigned int delta_time;
 335         u64 cputime_speedadj;
 336         int cpu_load;
 337         struct cpufreq_interactive_cpuinfo *pcpu =
 338                 &per_cpu(cpuinfo, data);
 339         unsigned int new_freq;
 340         unsigned int loadadjfreq;
 341         unsigned int index;
 342         unsigned long flags;
 343         bool boosted;
 344
 345         if (!down_read_trylock(&pcpu->enable_sem))
 346                 return;
 347         if (!pcpu->governor_enabled)
 348                 goto exit;
 349
 350         spin_lock_irqsave(&pcpu->load_lock, flags);
 351         now = update_load(data);
 352         delta_time = (unsigned int)(now - pcpu->cputime_speedadj_timestamp);
 353         cputime_speedadj = pcpu->cputime_speedadj;
 354         spin_unlock_irqrestore(&pcpu->load_lock, flags);
 355
 356         if (WARN_ON_ONCE(!delta_time))
 357                 goto rearm;
 358
 359         do_div(cputime_speedadj, delta_time);
 360         loadadjfreq = (unsigned int)cputime_speedadj * 100;
 361         cpu_load = loadadjfreq / pcpu->target_freq;
 362         boosted = boost_val || now < boostpulse_endtime;
 363
 364         if (cpu_load >= go_hispeed_load || boosted) {
 365                 if (pcpu->target_freq < hispeed_freq) {
 366                         new_freq = hispeed_freq;
 367                 } else {
 368                         new_freq = choose_freq(pcpu, loadadjfreq);
 369
 370                         if (new_freq < hispeed_freq)
 371                                 new_freq = hispeed_freq;
 372                 }
 373         } else {
 374                 new_freq = choose_freq(pcpu, loadadjfreq);
 375         }
 376
 377         if (pcpu->target_freq >= hispeed_freq &&
 378             new_freq > pcpu->target_freq &&
 379             now - pcpu->hispeed_validate_time <
 380             freq_to_above_hispeed_delay(pcpu->policy->cur)) {
 381                 trace_cpufreq_interactive_notyet(
 382                         data, cpu_load, pcpu->target_freq,
 383                         pcpu->policy->cur, new_freq);
 384                 goto rearm;
 385         }
 386
 387         pcpu->hispeed_validate_time = now;
 388
 389         if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
 390                                            new_freq, CPUFREQ_RELATION_L,
 391                                            &index)) {
 392                 pr_warn_once("timer %d: cpufreq_frequency_table_target error\n",
 393                              (int) data);
 394                 goto rearm;
 395         }
 396
 397         new_freq = pcpu->freq_table[index].frequency;
 398
 399         /*
 400          * Do not scale below floor_freq unless we have been at or above the
 401          * floor frequency for the minimum sample time since last validated.
 402          */
 403         if (new_freq < pcpu->floor_freq) {
 404                 if (now - pcpu->floor_validate_time < min_sample_time) {
 405                         trace_cpufreq_interactive_notyet(
 406                                 data, cpu_load, pcpu->target_freq,
 407                                 pcpu->policy->cur, new_freq);
 408                         goto rearm;
 409                 }
 410         }
 411
 412         /*
 413          * Update the timestamp for checking whether speed has been held at
 414          * or above the selected frequency for a minimum of min_sample_time,
 415          * if not boosted to hispeed_freq.  If boosted to hispeed_freq then we
 416          * allow the speed to drop as soon as the boostpulse duration expires
 417          * (or the indefinite boost is turned off).
 418          */
 419
 420         if (!boosted || new_freq > hispeed_freq) {
 421                 pcpu->floor_freq = new_freq;
 422                 pcpu->floor_validate_time = now;
 423         }
 424
 425         if (pcpu->target_freq == new_freq) {
 426                 trace_cpufreq_interactive_already(
 427                         data, cpu_load, pcpu->target_freq,
 428                         pcpu->policy->cur, new_freq);
 429                 goto rearm_if_notmax;
 430         }
 431
 432         trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
 433                                          pcpu->policy->cur, new_freq);
 434
 435         pcpu->target_freq = new_freq;
 436         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 437         cpumask_set_cpu(data, &speedchange_cpumask);
 438         spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 439         wake_up_process(speedchange_task);
 440
 441 rearm_if_notmax:
 442         /*
 443          * Already set max speed and don't see a need to change that,
 444          * wait until next idle to re-evaluate, don't need timer.
 445          */
 446         if (pcpu->target_freq == pcpu->policy->max)
 447                 goto exit;
 448
 449 rearm:
 450         if (!timer_pending(&pcpu->cpu_timer))
 451                 cpufreq_interactive_timer_resched(pcpu);
 452
 453 exit:
 454         up_read(&pcpu->enable_sem);
 455         return;
 456 }
 457
 458 static void cpufreq_interactive_idle_start(void)
 459 {
 460         struct cpufreq_interactive_cpuinfo *pcpu =
 461                 &per_cpu(cpuinfo, smp_processor_id());
 462         int pending;
 463
 464         if (!down_read_trylock(&pcpu->enable_sem))
 465                 return;
 466         if (!pcpu->governor_enabled) {
 467                 up_read(&pcpu->enable_sem);
 468                 return;
 469         }
 470
 471         pending = timer_pending(&pcpu->cpu_timer);
 472
 473         if (pcpu->target_freq != pcpu->policy->min) {
 474                 /*
 475                  * Entering idle while not at lowest speed.  On some
 476                  * platforms this can hold the other CPU(s) at that speed
 477                  * even though the CPU is idle. Set a timer to re-evaluate
 478                  * speed so this idle CPU doesn't hold the other CPUs above
 479                  * min indefinitely.  This should probably be a quirk of
 480                  * the CPUFreq driver.
 481                  */
 482                 if (!pending)
 483                         cpufreq_interactive_timer_resched(pcpu);
 484         }
 485
 486         up_read(&pcpu->enable_sem);
 487 }
 488
 489 static void cpufreq_interactive_idle_end(void)
 490 {
 491         struct cpufreq_interactive_cpuinfo *pcpu =
 492                 &per_cpu(cpuinfo, smp_processor_id());
 493
 494         if (!down_read_trylock(&pcpu->enable_sem))
 495                 return;
 496         if (!pcpu->governor_enabled) {
 497                 up_read(&pcpu->enable_sem);
 498                 return;
 499         }
 500
 501         /* Arm the timer for 1-2 ticks later if not already. */
 502         if (!timer_pending(&pcpu->cpu_timer)) {
 503                 cpufreq_interactive_timer_resched(pcpu);
 504         } else if (time_after_eq(jiffies, pcpu->cpu_timer.expires)) {
 505                 del_timer(&pcpu->cpu_timer);
 506                 del_timer(&pcpu->cpu_slack_timer);
 507                 cpufreq_interactive_timer(smp_processor_id());
 508         }
 509
 510         up_read(&pcpu->enable_sem);
 511 }
 512
 513 static int cpufreq_interactive_speedchange_task(void *data)
 514 {
 515         unsigned int cpu;
 516         cpumask_t tmp_mask;
 517         unsigned long flags;
 518         struct cpufreq_interactive_cpuinfo *pcpu;
 519
 520         while (1) {
 521                 set_current_state(TASK_INTERRUPTIBLE);
 522                 spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 523
 524                 if (cpumask_empty(&speedchange_cpumask)) {
 525                         spin_unlock_irqrestore(&speedchange_cpumask_lock,
 526                                                flags);
 527                         schedule();
 528
 529                         if (kthread_should_stop())
 530                                 break;
 531
 532                         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 533                 }
 534
 535                 set_current_state(TASK_RUNNING);
 536                 tmp_mask = speedchange_cpumask;
 537                 cpumask_clear(&speedchange_cpumask);
 538                 spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 539
 540                 for_each_cpu(cpu, &tmp_mask) {
 541                         unsigned int j;
 542                         unsigned int max_freq = 0;
 543
 544                         pcpu = &per_cpu(cpuinfo, cpu);
 545                         if (!down_read_trylock(&pcpu->enable_sem))
 546                                 continue;
 547                         if (!pcpu->governor_enabled) {
 548                                 up_read(&pcpu->enable_sem);
 549                                 continue;
 550                         }
 551
 552                         for_each_cpu(j, pcpu->policy->cpus) {
 553                                 struct cpufreq_interactive_cpuinfo *pjcpu =
 554                                         &per_cpu(cpuinfo, j);
 555
 556                                 if (pjcpu->target_freq > max_freq)
 557                                         max_freq = pjcpu->target_freq;
 558                         }
 559
 560                         if (max_freq != pcpu->policy->cur)
 561                                 __cpufreq_driver_target(pcpu->policy,
 562                                                         max_freq,
 563                                                         CPUFREQ_RELATION_H);
 564                         trace_cpufreq_interactive_setspeed(cpu,
 565                                                      pcpu->target_freq,
 566                                                      pcpu->policy->cur);
 567
 568                         up_read(&pcpu->enable_sem);
 569                 }
 570         }
 571
 572         return 0;
 573 }
 574
 575 static void cpufreq_interactive_boost(void)
 576 {
 577         int i;
 578         int anyboost = 0;
 579         unsigned long flags;
 580         struct cpufreq_interactive_cpuinfo *pcpu;
 581
 582         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 583
 584         for_each_online_cpu(i) {
 585                 pcpu = &per_cpu(cpuinfo, i);
 586
 587                 if (pcpu->target_freq < hispeed_freq) {
 588                         pcpu->target_freq = hispeed_freq;
 589                         cpumask_set_cpu(i, &speedchange_cpumask);
 590                         pcpu->hispeed_validate_time =
 591                                 ktime_to_us(ktime_get());
 592                         anyboost = 1;
 593                 }
 594
 595                 /*
 596                  * Set floor freq and (re)start timer for when last
 597                  * validated.
 598                  */
 599
 600                 pcpu->floor_freq = hispeed_freq;
 601                 pcpu->floor_validate_time = ktime_to_us(ktime_get());
 602         }
 603
 604         spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 605
 606         if (anyboost)
 607                 wake_up_process(speedchange_task);
 608 }
 609
 610 static int cpufreq_interactive_notifier(
 611         struct notifier_block *nb, unsigned long val, void *data)
 612 {
 613         struct cpufreq_freqs *freq = data;
 614         struct cpufreq_interactive_cpuinfo *pcpu;
 615         int cpu;
 616         unsigned long flags;
 617
 618         if (val == CPUFREQ_POSTCHANGE) {
 619                 pcpu = &per_cpu(cpuinfo, freq->cpu);
 620                 if (!down_read_trylock(&pcpu->enable_sem))
 621                         return 0;
 622                 if (!pcpu->governor_enabled) {
 623                         up_read(&pcpu->enable_sem);
 624                         return 0;
 625                 }
 626
 627                 for_each_cpu(cpu, pcpu->policy->cpus) {
 628                         struct cpufreq_interactive_cpuinfo *pjcpu =
 629                                 &per_cpu(cpuinfo, cpu);
 630                         spin_lock_irqsave(&pjcpu->load_lock, flags);
 631                         update_load(cpu);
 632                         spin_unlock_irqrestore(&pjcpu->load_lock, flags);
 633                 }
 634
 635                 up_read(&pcpu->enable_sem);
 636         }
 637         return 0;
 638 }
 639
 640 static struct notifier_block cpufreq_notifier_block = {
 641         .notifier_call = cpufreq_interactive_notifier,
 642 };
 643
 644 static unsigned int *get_tokenized_data(const char *buf, int *num_tokens)
 645 {
 646         const char *cp;
 647         int i;
 648         int ntokens = 1;
 649         unsigned int *tokenized_data;
 650
 651         cp = buf;
 652         while ((cp = strpbrk(cp + 1, " :")))
 653                 ntokens++;
 654
 655         if (!(ntokens & 0x1)) {
 656                 tokenized_data = ERR_PTR(-EINVAL);
 657                 goto err;
 658         }
 659
 660         tokenized_data = kmalloc(ntokens * sizeof(unsigned int), GFP_KERNEL);
 661         if (!tokenized_data) {
 662                 tokenized_data = ERR_PTR(-ENOMEM);
 663                 goto err;
 664         }
 665
 666         cp = buf;
 667         i = 0;
 668         while (i < ntokens) {
 669                 if (sscanf(cp, "%u", &tokenized_data[i++]) != 1) {
 670                         tokenized_data = ERR_PTR(-EINVAL);
 671                         goto err_kfree;
 672                 }
 673
 674                 cp = strpbrk(cp, " :");
 675                 if (!cp)
 676                         break;
 677                 cp++;
 678         }
 679
 680         if (i != ntokens) {
 681                 tokenized_data = ERR_PTR(-EINVAL);
 682                 goto err_kfree;
 683         }
 684
 685         *num_tokens = ntokens;
 686         return tokenized_data;
 687
 688 err_kfree:
 689         kfree(tokenized_data);
 690 err:
 691         return tokenized_data;
 692 }
 693
 694 static ssize_t show_target_loads(
 695         struct kobject *kobj, struct attribute *attr, char *buf)
 696 {
 697         int i;
 698         ssize_t ret = 0;
 699         unsigned long flags;
 700
 701         spin_lock_irqsave(&target_loads_lock, flags);
 702
 703         for (i = 0; i < ntarget_loads; i++)
 704                 ret += sprintf(buf + ret, "%u%s", target_loads[i],
 705                                i & 0x1 ? ":" : " ");
 706
 707         ret += sprintf(buf + ret, "\n");
 708         spin_unlock_irqrestore(&target_loads_lock, flags);
 709         return ret;
 710 }
 711
 712 static ssize_t store_target_loads(
 713         struct kobject *kobj, struct attribute *attr, const char *buf,
 714         size_t count)
 715 {
 716         int ntokens;
 717         unsigned int *new_target_loads = NULL;
 718         unsigned long flags;
 719
 720         new_target_loads = get_tokenized_data(buf, &ntokens);
 721         if (IS_ERR(new_target_loads))
 722                 return PTR_RET(new_target_loads);
 723
 724         spin_lock_irqsave(&target_loads_lock, flags);
 725         if (target_loads != default_target_loads)
 726                 kfree(target_loads);
 727         target_loads = new_target_loads;
 728         ntarget_loads = ntokens;
 729         spin_unlock_irqrestore(&target_loads_lock, flags);
 730         return count;
 731 }
 732
 733 static struct global_attr target_loads_attr =
 734         __ATTR(target_loads, S_IRUGO | S_IWUSR,
 735                 show_target_loads, store_target_loads);
 736
 737 static ssize_t show_above_hispeed_delay(
 738         struct kobject *kobj, struct attribute *attr, char *buf)
 739 {
 740         int i;
 741         ssize_t ret = 0;
 742         unsigned long flags;
 743
 744         spin_lock_irqsave(&above_hispeed_delay_lock, flags);
 745
 746         for (i = 0; i < nabove_hispeed_delay; i++)
 747                 ret += sprintf(buf + ret, "%u%s", above_hispeed_delay[i],
 748                                i & 0x1 ? ":" : " ");
 749
 750         ret += sprintf(buf + ret, "\n");
 751         spin_unlock_irqrestore(&above_hispeed_delay_lock, flags);
 752         return ret;
 753 }
 754
 755 static ssize_t store_above_hispeed_delay(
 756         struct kobject *kobj, struct attribute *attr, const char *buf,
 757         size_t count)
 758 {
 759         int ntokens;
 760         unsigned int *new_above_hispeed_delay = NULL;
 761         unsigned long flags;
 762
 763         new_above_hispeed_delay = get_tokenized_data(buf, &ntokens);
 764         if (IS_ERR(new_above_hispeed_delay))
 765                 return PTR_RET(new_above_hispeed_delay);
 766
 767         spin_lock_irqsave(&above_hispeed_delay_lock, flags);
 768         if (above_hispeed_delay != default_above_hispeed_delay)
 769                 kfree(above_hispeed_delay);
 770         above_hispeed_delay = new_above_hispeed_delay;
 771         nabove_hispeed_delay = ntokens;
 772         spin_unlock_irqrestore(&above_hispeed_delay_lock, flags);
 773         return count;
 774
 775 }
 776
 777 static struct global_attr above_hispeed_delay_attr =
 778         __ATTR(above_hispeed_delay, S_IRUGO | S_IWUSR,
 779                 show_above_hispeed_delay, store_above_hispeed_delay);
 780
 781 static ssize_t show_hispeed_freq(struct kobject *kobj,
 782                                  struct attribute *attr, char *buf)
 783 {
 784         return sprintf(buf, "%u\n", hispeed_freq);
 785 }
 786
 787 static ssize_t store_hispeed_freq(struct kobject *kobj,
 788                                   struct attribute *attr, const char *buf,
 789                                   size_t count)
 790 {
 791         int ret;
 792         long unsigned int val;
 793
 794         ret = strict_strtoul(buf, 0, &val);
 795         if (ret < 0)
 796                 return ret;
 797         hispeed_freq = val;
 798         return count;
 799 }
 800
 801 static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644,
 802                 show_hispeed_freq, store_hispeed_freq);
 803
 804
 805 static ssize_t show_go_hispeed_load(struct kobject *kobj,
 806                                      struct attribute *attr, char *buf)
 807 {
 808         return sprintf(buf, "%lu\n", go_hispeed_load);
 809 }
 810
 811 static ssize_t store_go_hispeed_load(struct kobject *kobj,
 812                         struct attribute *attr, const char *buf, size_t count)
 813 {
 814         int ret;
 815         unsigned long val;
 816
 817         ret = strict_strtoul(buf, 0, &val);
 818         if (ret < 0)
 819                 return ret;
 820         go_hispeed_load = val;
 821         return count;
 822 }
 823
 824 static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644,
 825                 show_go_hispeed_load, store_go_hispeed_load);
 826
 827 static ssize_t show_min_sample_time(struct kobject *kobj,
 828                                 struct attribute *attr, char *buf)
 829 {
 830         return sprintf(buf, "%lu\n", min_sample_time);
 831 }
 832
 833 static ssize_t store_min_sample_time(struct kobject *kobj,
 834                         struct attribute *attr, const char *buf, size_t count)
 835 {
 836         int ret;
 837         unsigned long val;
 838
 839         ret = strict_strtoul(buf, 0, &val);
 840         if (ret < 0)
 841                 return ret;
 842         min_sample_time = val;
 843         return count;
 844 }
 845
 846 static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
 847                 show_min_sample_time, store_min_sample_time);
 848
 849 static ssize_t show_timer_rate(struct kobject *kobj,
 850                         struct attribute *attr, char *buf)
 851 {
 852         return sprintf(buf, "%lu\n", timer_rate);
 853 }
 854
 855 static ssize_t store_timer_rate(struct kobject *kobj,
 856                         struct attribute *attr, const char *buf, size_t count)
 857 {
 858         int ret;
 859         unsigned long val;
 860
 861         ret = strict_strtoul(buf, 0, &val);
 862         if (ret < 0)
 863                 return ret;
 864         timer_rate = val;
 865         return count;
 866 }
 867
 868 static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
 869                 show_timer_rate, store_timer_rate);
 870
 871 static ssize_t show_timer_slack(
 872         struct kobject *kobj, struct attribute *attr, char *buf)
 873 {
 874         return sprintf(buf, "%d\n", timer_slack_val);
 875 }
 876
 877 static ssize_t store_timer_slack(
 878         struct kobject *kobj, struct attribute *attr, const char *buf,
 879         size_t count)
 880 {
 881         int ret;
 882         unsigned long val;
 883
 884         ret = kstrtol(buf, 10, &val);
 885         if (ret < 0)
 886                 return ret;
 887
 888         timer_slack_val = val;
 889         return count;
 890 }
 891
 892 define_one_global_rw(timer_slack);
 893
 894 static ssize_t show_boost(struct kobject *kobj, struct attribute *attr,
 895                           char *buf)
 896 {
 897         return sprintf(buf, "%d\n", boost_val);
 898 }
 899
 900 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 901                            const char *buf, size_t count)
 902 {
 903         int ret;
 904         unsigned long val;
 905
 906         ret = kstrtoul(buf, 0, &val);
 907         if (ret < 0)
 908                 return ret;
 909
 910         boost_val = val;
 911
 912         if (boost_val) {
 913                 trace_cpufreq_interactive_boost("on");
 914                 cpufreq_interactive_boost();
 915         } else {
 916                 trace_cpufreq_interactive_unboost("off");
 917         }
 918
 919         return count;
 920 }
 921
 922 define_one_global_rw(boost);
 923
 924 static ssize_t store_boostpulse(struct kobject *kobj, struct attribute *attr,
 925                                 const char *buf, size_t count)
 926 {
 927         int ret;
 928         unsigned long val;
 929
 930         ret = kstrtoul(buf, 0, &val);
 931         if (ret < 0)
 932                 return ret;
 933
 934         boostpulse_endtime = ktime_to_us(ktime_get()) + boostpulse_duration_val;
 935         trace_cpufreq_interactive_boost("pulse");
 936         cpufreq_interactive_boost();
 937         return count;
 938 }
 939
 940 static struct global_attr boostpulse =
 941         __ATTR(boostpulse, 0200, NULL, store_boostpulse);
 942
 943 static ssize_t show_boostpulse_duration(
 944         struct kobject *kobj, struct attribute *attr, char *buf)
 945 {
 946         return sprintf(buf, "%d\n", boostpulse_duration_val);
 947 }
 948
 949 static ssize_t store_boostpulse_duration(
 950         struct kobject *kobj, struct attribute *attr, const char *buf,
 951         size_t count)
 952 {
 953         int ret;
 954         unsigned long val;
 955
 956         ret = kstrtoul(buf, 0, &val);
 957         if (ret < 0)
 958                 return ret;
 959
 960         boostpulse_duration_val = val;
 961         return count;
 962 }
 963
 964 define_one_global_rw(boostpulse_duration);
 965
 966 static ssize_t show_io_is_busy(struct kobject *kobj,
 967                         struct attribute *attr, char *buf)
 968 {
 969         return sprintf(buf, "%u\n", io_is_busy);
 970 }
 971
 972 static ssize_t store_io_is_busy(struct kobject *kobj,
 973                         struct attribute *attr, const char *buf, size_t count)
 974 {
 975         int ret;
 976         unsigned long val;
 977
 978         ret = kstrtoul(buf, 0, &val);
 979         if (ret < 0)
 980                 return ret;
 981         io_is_busy = val;
 982         return count;
 983 }
 984
 985 static struct global_attr io_is_busy_attr = __ATTR(io_is_busy, 0644,
 986                 show_io_is_busy, store_io_is_busy);
 987
 988 static struct attribute *interactive_attributes[] = {
 989         &target_loads_attr.attr,
 990         &above_hispeed_delay_attr.attr,
 991         &hispeed_freq_attr.attr,
 992         &go_hispeed_load_attr.attr,
 993         &min_sample_time_attr.attr,
 994         &timer_rate_attr.attr,
 995         &timer_slack.attr,
 996         &boost.attr,
 997         &boostpulse.attr,
 998         &boostpulse_duration.attr,
 999         &io_is_busy_attr.attr,
1000         NULL,
1001 };
1002
1003 static struct attribute_group interactive_attr_group = {
1004         .attrs = interactive_attributes,
1005         .name = "interactive",
1006 };
1007
1008 static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
1009                                              unsigned long val,
1010                                              void *data)
1011 {
1012         switch (val) {
1013         case IDLE_START:
1014                 cpufreq_interactive_idle_start();
1015                 break;
1016         case IDLE_END:
1017                 cpufreq_interactive_idle_end();
1018                 break;
1019         }
1020
1021         return 0;
1022 }
1023
1024 static struct notifier_block cpufreq_interactive_idle_nb = {
1025         .notifier_call = cpufreq_interactive_idle_notifier,
1026 };
1027
1028 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
1029                 unsigned int event)
1030 {
1031         int rc;
1032         unsigned int j;
1033         struct cpufreq_interactive_cpuinfo *pcpu;
1034         struct cpufreq_frequency_table *freq_table;
1035
1036         switch (event) {
1037         case CPUFREQ_GOV_START:
1038                 if (!cpu_online(policy->cpu))
1039                         return -EINVAL;
1040
1041                 mutex_lock(&gov_lock);
1042
1043                 freq_table =
1044                         cpufreq_frequency_get_table(policy->cpu);
1045                 if (!hispeed_freq)
1046                         hispeed_freq = policy->max;
1047
1048                 for_each_cpu(j, policy->cpus) {
1049                         unsigned long expires;
1050
1051                         pcpu = &per_cpu(cpuinfo, j);
1052                         pcpu->policy = policy;
1053                         pcpu->target_freq = policy->cur;
1054                         pcpu->freq_table = freq_table;
1055                         pcpu->floor_freq = pcpu->target_freq;
1056                         pcpu->floor_validate_time =
1057                                 ktime_to_us(ktime_get());
1058                         pcpu->hispeed_validate_time =
1059                                 pcpu->floor_validate_time;
1060                         down_write(&pcpu->enable_sem);
1061                         expires = jiffies + usecs_to_jiffies(timer_rate);
1062                         pcpu->cpu_timer.expires = expires;
1063                         add_timer_on(&pcpu->cpu_timer, j);
1064                         if (timer_slack_val >= 0) {
1065                                 expires += usecs_to_jiffies(timer_slack_val);
1066                                 pcpu->cpu_slack_timer.expires = expires;
1067                                 add_timer_on(&pcpu->cpu_slack_timer, j);
1068                         }
1069                         pcpu->governor_enabled = 1;
1070                         up_write(&pcpu->enable_sem);
1071                 }
1072
1073                 /*
1074                  * Do not register the idle hook and create sysfs
1075                  * entries if we have already done so.
1076                  */
1077                 if (++active_count > 1) {
1078                         mutex_unlock(&gov_lock);
1079                         return 0;
1080                 }
1081
1082                 rc = sysfs_create_group(cpufreq_global_kobject,
1083                                 &interactive_attr_group);
1084                 if (rc) {
1085                         mutex_unlock(&gov_lock);
1086                         return rc;
1087                 }
1088
1089                 idle_notifier_register(&cpufreq_interactive_idle_nb);
1090                 cpufreq_register_notifier(
1091                         &cpufreq_notifier_block, CPUFREQ_TRANSITION_NOTIFIER);
1092                 mutex_unlock(&gov_lock);
1093                 break;
1094
1095         case CPUFREQ_GOV_STOP:
1096                 mutex_lock(&gov_lock);
1097                 for_each_cpu(j, policy->cpus) {
1098                         pcpu = &per_cpu(cpuinfo, j);
1099                         down_write(&pcpu->enable_sem);
1100                         pcpu->governor_enabled = 0;
1101                         del_timer_sync(&pcpu->cpu_timer);
1102                         del_timer_sync(&pcpu->cpu_slack_timer);
1103                         up_write(&pcpu->enable_sem);
1104                 }
1105
1106                 if (--active_count > 0) {
1107                         mutex_unlock(&gov_lock);
1108                         return 0;
1109                 }
1110
1111                 cpufreq_unregister_notifier(
1112                         &cpufreq_notifier_block, CPUFREQ_TRANSITION_NOTIFIER);
1113                 idle_notifier_unregister(&cpufreq_interactive_idle_nb);
1114                 sysfs_remove_group(cpufreq_global_kobject,
1115                                 &interactive_attr_group);
1116                 mutex_unlock(&gov_lock);
1117
1118                 break;
1119
1120         case CPUFREQ_GOV_LIMITS:
1121                 if (policy->max < policy->cur)
1122                         __cpufreq_driver_target(policy,
1123                                         policy->max, CPUFREQ_RELATION_H);
1124                 else if (policy->min > policy->cur)
1125                         __cpufreq_driver_target(policy,
1126                                         policy->min, CPUFREQ_RELATION_L);
1127                 break;
1128         }
1129         return 0;
1130 }
1131
1132 static void cpufreq_interactive_nop_timer(unsigned long data)
1133 {
1134 }
1135
1136 static int __init cpufreq_interactive_init(void)
1137 {
1138         unsigned int i;
1139         struct cpufreq_interactive_cpuinfo *pcpu;
1140         struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
1141
1142         /* Initalize per-cpu timers */
1143         for_each_possible_cpu(i) {
1144                 pcpu = &per_cpu(cpuinfo, i);
1145                 init_timer_deferrable(&pcpu->cpu_timer);
1146                 pcpu->cpu_timer.function = cpufreq_interactive_timer;
1147                 pcpu->cpu_timer.data = i;
1148                 init_timer(&pcpu->cpu_slack_timer);
1149                 pcpu->cpu_slack_timer.function = cpufreq_interactive_nop_timer;
1150                 spin_lock_init(&pcpu->load_lock);
1151                 init_rwsem(&pcpu->enable_sem);
1152         }
1153
1154         spin_lock_init(&target_loads_lock);
1155         spin_lock_init(&speedchange_cpumask_lock);
1156         mutex_init(&gov_lock);
1157         speedchange_task =
1158                 kthread_create(cpufreq_interactive_speedchange_task, NULL,
1159                                "cfinteractive");
1160         if (IS_ERR(speedchange_task))
1161                 return PTR_ERR(speedchange_task);
1162
1163         sched_setscheduler_nocheck(speedchange_task, SCHED_FIFO, &param);
1164         get_task_struct(speedchange_task);
1165
1166         /* NB: wake up so the thread does not look hung to the freezer */
1167         wake_up_process(speedchange_task);
1168
1169         return cpufreq_register_governor(&cpufreq_gov_interactive);
1170 }
1171
1172 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
1173 fs_initcall(cpufreq_interactive_init);
1174 #else
1175 module_init(cpufreq_interactive_init);
1176 #endif
1177
1178 static void __exit cpufreq_interactive_exit(void)
1179 {
1180         cpufreq_unregister_governor(&cpufreq_gov_interactive);
1181         kthread_stop(speedchange_task);
1182         put_task_struct(speedchange_task);
1183 }
1184
1185 module_exit(cpufreq_interactive_exit);
1186
1187 MODULE_AUTHOR("Mike Chan <mike@android.com>");
1188 MODULE_DESCRIPTION("'cpufreq_interactive' - A cpufreq governor for "
1189         "Latency sensitive workloads");
1190 MODULE_LICENSE("GPL");