drivers/cpufreq/cpufreq_interactive.c

   1 /*
   2  * drivers/cpufreq/cpufreq_interactive.c
   3  *
   4  * Copyright (C) 2010 Google, Inc.
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * Author: Mike Chan (mike@android.com)
  16  *
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/cpumask.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/module.h>
  23 #include <linux/moduleparam.h>
  24 #include <linux/rwsem.h>
  25 #include <linux/sched.h>
  26 #include <linux/sched/rt.h>
  27 #include <linux/tick.h>
  28 #include <linux/time.h>
  29 #include <linux/timer.h>
  30 #include <linux/workqueue.h>
  31 #include <linux/kthread.h>
  32 #include <linux/slab.h>
  33 #include <linux/kernel_stat.h>
  34 #include <asm/cputime.h>
  35
  36 #define CREATE_TRACE_POINTS
  37 #include <trace/events/cpufreq_interactive.h>
  38
  39 static int active_count;
  40
  41 struct cpufreq_interactive_cpuinfo {
  42         struct timer_list cpu_timer;
  43         struct timer_list cpu_slack_timer;
  44         spinlock_t load_lock; /* protects the next 4 fields */
  45         u64 time_in_idle;
  46         u64 time_in_idle_timestamp;
  47         u64 cputime_speedadj;
  48         u64 cputime_speedadj_timestamp;
  49         struct cpufreq_policy *policy;
  50         struct cpufreq_frequency_table *freq_table;
  51         unsigned int target_freq;
  52         unsigned int floor_freq;
  53         u64 floor_validate_time;
  54         u64 hispeed_validate_time;
  55         struct rw_semaphore enable_sem;
  56         int governor_enabled;
  57 };
  58
  59 static DEFINE_PER_CPU(struct cpufreq_interactive_cpuinfo, cpuinfo);
  60
  61 /* realtime thread handles frequency scaling */
  62 static struct task_struct *speedchange_task;
  63 static cpumask_t speedchange_cpumask;
  64 static spinlock_t speedchange_cpumask_lock;
  65 static struct mutex gov_lock;
  66
  67 /* Hi speed to bump to from lo speed when load burst (default max) */
  68 static unsigned int hispeed_freq;
  69
  70 /* Go to hi speed when CPU load at or above this value. */
  71 #define DEFAULT_GO_HISPEED_LOAD 99
  72 static unsigned long go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
  73
  74 /* Target load.  Lower values result in higher CPU speeds. */
  75 #define DEFAULT_TARGET_LOAD 90
  76 static unsigned int default_target_loads[] = {DEFAULT_TARGET_LOAD};
  77 static spinlock_t target_loads_lock;
  78 static unsigned int *target_loads = default_target_loads;
  79 static int ntarget_loads = ARRAY_SIZE(default_target_loads);
  80
  81 /*
  82  * The minimum amount of time to spend at a frequency before we can ramp down.
  83  */
  84 #define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
  85 static unsigned long min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
  86
  87 /*
  88  * The sample rate of the timer used to increase frequency
  89  */
  90 #define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC)
  91 static unsigned long timer_rate = DEFAULT_TIMER_RATE;
  92
  93 /*
  94  * Wait this long before raising speed above hispeed, by default a single
  95  * timer interval.
  96  */
  97 #define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE
  98 static unsigned int default_above_hispeed_delay[] = {
  99         DEFAULT_ABOVE_HISPEED_DELAY };
 100 static spinlock_t above_hispeed_delay_lock;
 101 static unsigned int *above_hispeed_delay = default_above_hispeed_delay;
 102 static int nabove_hispeed_delay = ARRAY_SIZE(default_above_hispeed_delay);
 103
 104 /* Non-zero means indefinite speed boost active */
 105 static int boost_val;
 106 /* Duration of a boot pulse in usecs */
 107 static int boostpulse_duration_val = DEFAULT_MIN_SAMPLE_TIME;
 108 /* End time of boost pulse in ktime converted to usecs */
 109 static u64 boostpulse_endtime;
 110
 111 /*
 112  * Max additional time to wait in idle, beyond timer_rate, at speeds above
 113  * minimum before wakeup to reduce speed, or -1 if unnecessary.
 114  */
 115 #define DEFAULT_TIMER_SLACK (4 * DEFAULT_TIMER_RATE)
 116 static int timer_slack_val = DEFAULT_TIMER_SLACK;
 117
 118 static bool io_is_busy;
 119
 120 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 121                 unsigned int event);
 122
 123 #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 124 static
 125 #endif
 126 struct cpufreq_governor cpufreq_gov_interactive = {
 127         .name = "interactive",
 128         .governor = cpufreq_governor_interactive,
 129         .max_transition_latency = 10000000,
 130         .owner = THIS_MODULE,
 131 };
 132
 133 static inline cputime64_t get_cpu_idle_time_jiffy(unsigned int cpu,
 134                                                   cputime64_t *wall)
 135 {
 136         u64 idle_time;
 137         u64 cur_wall_time;
 138         u64 busy_time;
 139
 140         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 141
 142         busy_time  = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 143         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 144         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 145         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 146         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 147         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 148
 149         idle_time = cur_wall_time - busy_time;
 150         if (wall)
 151                 *wall = jiffies_to_usecs(cur_wall_time);
 152
 153         return jiffies_to_usecs(idle_time);
 154 }
 155
 156 static inline cputime64_t get_cpu_idle_time(unsigned int cpu,
 157                                             cputime64_t *wall)
 158 {
 159         u64 idle_time = get_cpu_idle_time_us(cpu, wall);
 160
 161         if (idle_time == -1ULL)
 162                 idle_time = get_cpu_idle_time_jiffy(cpu, wall);
 163         else if (!io_is_busy)
 164                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 165
 166         return idle_time;
 167 }
 168
 169 static void cpufreq_interactive_timer_resched(
 170         struct cpufreq_interactive_cpuinfo *pcpu)
 171 {
 172         unsigned long expires = jiffies + usecs_to_jiffies(timer_rate);
 173         unsigned long flags;
 174
 175         mod_timer_pinned(&pcpu->cpu_timer, expires);
 176         if (timer_slack_val >= 0 && pcpu->target_freq > pcpu->policy->min) {
 177                 expires += usecs_to_jiffies(timer_slack_val);
 178                 mod_timer_pinned(&pcpu->cpu_slack_timer, expires);
 179         }
 180
 181         spin_lock_irqsave(&pcpu->load_lock, flags);
 182         pcpu->time_in_idle =
 183                 get_cpu_idle_time(smp_processor_id(),
 184                                      &pcpu->time_in_idle_timestamp);
 185         pcpu->cputime_speedadj = 0;
 186         pcpu->cputime_speedadj_timestamp = pcpu->time_in_idle_timestamp;
 187         spin_unlock_irqrestore(&pcpu->load_lock, flags);
 188 }
 189
 190 static unsigned int freq_to_above_hispeed_delay(unsigned int freq)
 191 {
 192         int i;
 193         unsigned int ret;
 194         unsigned long flags;
 195
 196         spin_lock_irqsave(&above_hispeed_delay_lock, flags);
 197
 198         for (i = 0; i < nabove_hispeed_delay - 1 &&
 199                         freq >= above_hispeed_delay[i+1]; i += 2)
 200                 ;
 201
 202         ret = above_hispeed_delay[i];
 203         spin_unlock_irqrestore(&above_hispeed_delay_lock, flags);
 204         return ret;
 205 }
 206
 207 static unsigned int freq_to_targetload(unsigned int freq)
 208 {
 209         int i;
 210         unsigned int ret;
 211         unsigned long flags;
 212
 213         spin_lock_irqsave(&target_loads_lock, flags);
 214
 215         for (i = 0; i < ntarget_loads - 1 && freq >= target_loads[i+1]; i += 2)
 216                 ;
 217
 218         ret = target_loads[i];
 219         spin_unlock_irqrestore(&target_loads_lock, flags);
 220         return ret;
 221 }
 222
 223 /*
 224  * If increasing frequencies never map to a lower target load then
 225  * choose_freq() will find the minimum frequency that does not exceed its
 226  * target load given the current load.
 227  */
 228
 229 static unsigned int choose_freq(
 230         struct cpufreq_interactive_cpuinfo *pcpu, unsigned int loadadjfreq)
 231 {
 232         unsigned int freq = pcpu->policy->cur;
 233         unsigned int prevfreq, freqmin, freqmax;
 234         unsigned int tl;
 235         int index;
 236
 237         freqmin = 0;
 238         freqmax = UINT_MAX;
 239
 240         do {
 241                 prevfreq = freq;
 242                 tl = freq_to_targetload(freq);
 243
 244                 /*
 245                  * Find the lowest frequency where the computed load is less
 246                  * than or equal to the target load.
 247                  */
 248
 249                 cpufreq_frequency_table_target(
 250                         pcpu->policy, pcpu->freq_table, loadadjfreq / tl,
 251                         CPUFREQ_RELATION_L, &index);
 252                 freq = pcpu->freq_table[index].frequency;
 253
 254                 if (freq > prevfreq) {
 255                         /* The previous frequency is too low. */
 256                         freqmin = prevfreq;
 257
 258                         if (freq >= freqmax) {
 259                                 /*
 260                                  * Find the highest frequency that is less
 261                                  * than freqmax.
 262                                  */
 263                                 cpufreq_frequency_table_target(
 264                                         pcpu->policy, pcpu->freq_table,
 265                                         freqmax - 1, CPUFREQ_RELATION_H,
 266                                         &index);
 267                                 freq = pcpu->freq_table[index].frequency;
 268
 269                                 if (freq == freqmin) {
 270                                         /*
 271                                          * The first frequency below freqmax
 272                                          * has already been found to be too
 273                                          * low.  freqmax is the lowest speed
 274                                          * we found that is fast enough.
 275                                          */
 276                                         freq = freqmax;
 277                                         break;
 278                                 }
 279                         }
 280                 } else if (freq < prevfreq) {
 281                         /* The previous frequency is high enough. */
 282                         freqmax = prevfreq;
 283
 284                         if (freq <= freqmin) {
 285                                 /*
 286                                  * Find the lowest frequency that is higher
 287                                  * than freqmin.
 288                                  */
 289                                 cpufreq_frequency_table_target(
 290                                         pcpu->policy, pcpu->freq_table,
 291                                         freqmin + 1, CPUFREQ_RELATION_L,
 292                                         &index);
 293                                 freq = pcpu->freq_table[index].frequency;
 294
 295                                 /*
 296                                  * If freqmax is the first frequency above
 297                                  * freqmin then we have already found that
 298                                  * this speed is fast enough.
 299                                  */
 300                                 if (freq == freqmax)
 301                                         break;
 302                         }
 303                 }
 304
 305                 /* If same frequency chosen as previous then done. */
 306         } while (freq != prevfreq);
 307
 308         return freq;
 309 }
 310
 311 static u64 update_load(int cpu)
 312 {
 313         struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, cpu);
 314         u64 now;
 315         u64 now_idle;
 316         unsigned int delta_idle;
 317         unsigned int delta_time;
 318         u64 active_time;
 319
 320         now_idle = get_cpu_idle_time(cpu, &now);
 321         delta_idle = (unsigned int)(now_idle - pcpu->time_in_idle);
 322         delta_time = (unsigned int)(now - pcpu->time_in_idle_timestamp);
 323         active_time = delta_time - delta_idle;
 324         pcpu->cputime_speedadj += active_time * pcpu->policy->cur;
 325
 326         pcpu->time_in_idle = now_idle;
 327         pcpu->time_in_idle_timestamp = now;
 328         return now;
 329 }
 330
 331 static void cpufreq_interactive_timer(unsigned long data)
 332 {
 333         u64 now;
 334         unsigned int delta_time;
 335         u64 cputime_speedadj;
 336         int cpu_load;
 337         struct cpufreq_interactive_cpuinfo *pcpu =
 338                 &per_cpu(cpuinfo, data);
 339         unsigned int new_freq;
 340         unsigned int loadadjfreq;
 341         unsigned int index;
 342         unsigned long flags;
 343         bool boosted;
 344
 345         if (!down_read_trylock(&pcpu->enable_sem))
 346                 return;
 347         if (!pcpu->governor_enabled)
 348                 goto exit;
 349
 350         spin_lock_irqsave(&pcpu->load_lock, flags);
 351         now = update_load(data);
 352         delta_time = (unsigned int)(now - pcpu->cputime_speedadj_timestamp);
 353         cputime_speedadj = pcpu->cputime_speedadj;
 354         spin_unlock_irqrestore(&pcpu->load_lock, flags);
 355
 356         if (WARN_ON_ONCE(!delta_time))
 357                 goto rearm;
 358
 359         do_div(cputime_speedadj, delta_time);
 360         loadadjfreq = (unsigned int)cputime_speedadj * 100;
 361         cpu_load = loadadjfreq / pcpu->target_freq;
 362         boosted = boost_val || now < boostpulse_endtime;
 363
 364         if (cpu_load >= go_hispeed_load || boosted) {
 365                 if (pcpu->target_freq < hispeed_freq) {
 366                         new_freq = hispeed_freq;
 367                 } else {
 368                         new_freq = choose_freq(pcpu, loadadjfreq);
 369
 370                         if (new_freq < hispeed_freq)
 371                                 new_freq = hispeed_freq;
 372                 }
 373         } else {
 374                 new_freq = choose_freq(pcpu, loadadjfreq);
 375         }
 376
 377         if (pcpu->target_freq >= hispeed_freq &&
 378             new_freq > pcpu->target_freq &&
 379             now - pcpu->hispeed_validate_time <
 380             freq_to_above_hispeed_delay(pcpu->policy->cur)) {
 381                 trace_cpufreq_interactive_notyet(
 382                         data, cpu_load, pcpu->target_freq,
 383                         pcpu->policy->cur, new_freq);
 384                 goto rearm;
 385         }
 386
 387         pcpu->hispeed_validate_time = now;
 388
 389         if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
 390                                            new_freq, CPUFREQ_RELATION_L,
 391                                            &index)) {
 392                 pr_warn_once("timer %d: cpufreq_frequency_table_target error\n",
 393                              (int) data);
 394                 goto rearm;
 395         }
 396
 397         new_freq = pcpu->freq_table[index].frequency;
 398
 399         /*
 400          * Do not scale below floor_freq unless we have been at or above the
 401          * floor frequency for the minimum sample time since last validated.
 402          */
 403         if (new_freq < pcpu->floor_freq) {
 404                 if (now - pcpu->floor_validate_time < min_sample_time) {
 405                         trace_cpufreq_interactive_notyet(
 406                                 data, cpu_load, pcpu->target_freq,
 407                                 pcpu->policy->cur, new_freq);
 408                         goto rearm;
 409                 }
 410         }
 411
 412         /*
 413          * Update the timestamp for checking whether speed has been held at
 414          * or above the selected frequency for a minimum of min_sample_time,
 415          * if not boosted to hispeed_freq.  If boosted to hispeed_freq then we
 416          * allow the speed to drop as soon as the boostpulse duration expires
 417          * (or the indefinite boost is turned off).
 418          */
 419
 420         if (!boosted || new_freq > hispeed_freq) {
 421                 pcpu->floor_freq = new_freq;
 422                 pcpu->floor_validate_time = now;
 423         }
 424
 425         if (pcpu->target_freq == new_freq) {
 426                 trace_cpufreq_interactive_already(
 427                         data, cpu_load, pcpu->target_freq,
 428                         pcpu->policy->cur, new_freq);
 429                 goto rearm_if_notmax;
 430         }
 431
 432         trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
 433                                          pcpu->policy->cur, new_freq);
 434
 435         pcpu->target_freq = new_freq;
 436         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 437         cpumask_set_cpu(data, &speedchange_cpumask);
 438         spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 439         wake_up_process(speedchange_task);
 440
 441 rearm_if_notmax:
 442         /*
 443          * Already set max speed and don't see a need to change that,
 444          * wait until next idle to re-evaluate, don't need timer.
 445          */
 446         if (pcpu->target_freq == pcpu->policy->max)
 447                 goto exit;
 448
 449 rearm:
 450         if (!timer_pending(&pcpu->cpu_timer))
 451                 cpufreq_interactive_timer_resched(pcpu);
 452
 453 exit:
 454         up_read(&pcpu->enable_sem);
 455         return;
 456 }
 457
 458 static void cpufreq_interactive_idle_start(void)
 459 {
 460         struct cpufreq_interactive_cpuinfo *pcpu =
 461                 &per_cpu(cpuinfo, smp_processor_id());
 462         int pending;
 463
 464         if (!down_read_trylock(&pcpu->enable_sem))
 465                 return;
 466         if (!pcpu->governor_enabled) {
 467                 up_read(&pcpu->enable_sem);
 468                 return;
 469         }
 470
 471         pending = timer_pending(&pcpu->cpu_timer);
 472
 473         if (pcpu->target_freq != pcpu->policy->min) {
 474                 /*
 475                  * Entering idle while not at lowest speed.  On some
 476                  * platforms this can hold the other CPU(s) at that speed
 477                  * even though the CPU is idle. Set a timer to re-evaluate
 478                  * speed so this idle CPU doesn't hold the other CPUs above
 479                  * min indefinitely.  This should probably be a quirk of
 480                  * the CPUFreq driver.
 481                  */
 482                 if (!pending)
 483                         cpufreq_interactive_timer_resched(pcpu);
 484         }
 485
 486         up_read(&pcpu->enable_sem);
 487 }
 488
 489 static void cpufreq_interactive_idle_end(void)
 490 {
 491         struct cpufreq_interactive_cpuinfo *pcpu =
 492                 &per_cpu(cpuinfo, smp_processor_id());
 493
 494         if (!down_read_trylock(&pcpu->enable_sem))
 495                 return;
 496         if (!pcpu->governor_enabled) {
 497                 up_read(&pcpu->enable_sem);
 498                 return;
 499         }
 500
 501         /* Arm the timer for 1-2 ticks later if not already. */
 502         if (!timer_pending(&pcpu->cpu_timer)) {
 503                 cpufreq_interactive_timer_resched(pcpu);
 504         } else if (time_after_eq(jiffies, pcpu->cpu_timer.expires)) {
 505                 del_timer(&pcpu->cpu_timer);
 506                 del_timer(&pcpu->cpu_slack_timer);
 507                 cpufreq_interactive_timer(smp_processor_id());
 508         }
 509
 510         up_read(&pcpu->enable_sem);
 511 }
 512
 513 static int cpufreq_interactive_speedchange_task(void *data)
 514 {
 515         unsigned int cpu;
 516         cpumask_t tmp_mask;
 517         unsigned long flags;
 518         struct cpufreq_interactive_cpuinfo *pcpu;
 519
 520         while (1) {
 521                 set_current_state(TASK_INTERRUPTIBLE);
 522                 spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 523
 524                 if (cpumask_empty(&speedchange_cpumask)) {
 525                         spin_unlock_irqrestore(&speedchange_cpumask_lock,
 526                                                flags);
 527                         schedule();
 528
 529                         if (kthread_should_stop())
 530                                 break;
 531
 532                         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 533                 }
 534
 535                 set_current_state(TASK_RUNNING);
 536                 tmp_mask = speedchange_cpumask;
 537                 cpumask_clear(&speedchange_cpumask);
 538                 spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 539
 540                 for_each_cpu(cpu, &tmp_mask) {
 541                         unsigned int j;
 542                         unsigned int max_freq = 0;
 543
 544                         pcpu = &per_cpu(cpuinfo, cpu);
 545                         if (!down_read_trylock(&pcpu->enable_sem))
 546                                 continue;
 547                         if (!pcpu->governor_enabled) {
 548                                 up_read(&pcpu->enable_sem);
 549                                 continue;
 550                         }
 551
 552                         for_each_cpu(j, pcpu->policy->cpus) {
 553                                 struct cpufreq_interactive_cpuinfo *pjcpu =
 554                                         &per_cpu(cpuinfo, j);
 555
 556                                 if (pjcpu->target_freq > max_freq)
 557                                         max_freq = pjcpu->target_freq;
 558                         }
 559
 560                         if (max_freq != pcpu->policy->cur)
 561                                 __cpufreq_driver_target(pcpu->policy,
 562                                                         max_freq,
 563                                                         CPUFREQ_RELATION_H);
 564                         trace_cpufreq_interactive_setspeed(cpu,
 565                                                      pcpu->target_freq,
 566                                                      pcpu->policy->cur);
 567
 568                         up_read(&pcpu->enable_sem);
 569                 }
 570         }
 571
 572         return 0;
 573 }
 574
 575 static void cpufreq_interactive_boost(void)
 576 {
 577         int i;
 578         int anyboost = 0;
 579         unsigned long flags;
 580         struct cpufreq_interactive_cpuinfo *pcpu;
 581
 582         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 583
 584         for_each_online_cpu(i) {
 585                 pcpu = &per_cpu(cpuinfo, i);
 586
 587                 if (pcpu->target_freq < hispeed_freq) {
 588                         pcpu->target_freq = hispeed_freq;
 589                         cpumask_set_cpu(i, &speedchange_cpumask);
 590                         pcpu->hispeed_validate_time =
 591                                 ktime_to_us(ktime_get());
 592                         anyboost = 1;
 593                 }
 594
 595                 /*
 596                  * Set floor freq and (re)start timer for when last
 597                  * validated.
 598                  */
 599
 600                 pcpu->floor_freq = hispeed_freq;
 601                 pcpu->floor_validate_time = ktime_to_us(ktime_get());
 602         }
 603
 604         spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 605
 606         if (anyboost)
 607                 wake_up_process(speedchange_task);
 608 }
 609
 610 static int cpufreq_interactive_notifier(
 611         struct notifier_block *nb, unsigned long val, void *data)
 612 {
 613         struct cpufreq_freqs *freq = data;
 614         struct cpufreq_interactive_cpuinfo *pcpu;
 615         int cpu;
 616         unsigned long flags;
 617
 618         if (val == CPUFREQ_POSTCHANGE) {
 619                 pcpu = &per_cpu(cpuinfo, freq->cpu);
 620                 if (!down_read_trylock(&pcpu->enable_sem))
 621                         return 0;
 622                 if (!pcpu->governor_enabled) {
 623                         up_read(&pcpu->enable_sem);
 624                         return 0;
 625                 }
 626
 627                 for_each_cpu(cpu, pcpu->policy->cpus) {
 628                         struct cpufreq_interactive_cpuinfo *pjcpu =
 629                                 &per_cpu(cpuinfo, cpu);
 630                         spin_lock_irqsave(&pjcpu->load_lock, flags);
 631                         update_load(cpu);
 632                         spin_unlock_irqrestore(&pjcpu->load_lock, flags);
 633                 }
 634
 635                 up_read(&pcpu->enable_sem);
 636         }
 637         return 0;
 638 }
 639
 640 static struct notifier_block cpufreq_notifier_block = {
 641         .notifier_call = cpufreq_interactive_notifier,
 642 };
 643
 644 static unsigned int *get_tokenized_data(const char *buf, int *num_tokens)
 645 {
 646         const char *cp;
 647         int i;
 648         int ntokens = 1;
 649         unsigned int *tokenized_data;
 650         int err = -EINVAL;
 651
 652         cp = buf;
 653         while ((cp = strpbrk(cp + 1, " :")))
 654                 ntokens++;
 655
 656         if (!(ntokens & 0x1))
 657                 goto err;
 658
 659         tokenized_data = kmalloc(ntokens * sizeof(unsigned int), GFP_KERNEL);
 660         if (!tokenized_data) {
 661                 err = -ENOMEM;
 662                 goto err;
 663         }
 664
 665         cp = buf;
 666         i = 0;
 667         while (i < ntokens) {
 668                 if (sscanf(cp, "%u", &tokenized_data[i++]) != 1)
 669                         goto err_kfree;
 670
 671                 cp = strpbrk(cp, " :");
 672                 if (!cp)
 673                         break;
 674                 cp++;
 675         }
 676
 677         if (i != ntokens)
 678                 goto err_kfree;
 679
 680         *num_tokens = ntokens;
 681         return tokenized_data;
 682
 683 err_kfree:
 684         kfree(tokenized_data);
 685 err:
 686         return ERR_PTR(err);
 687 }
 688
 689 static ssize_t show_target_loads(
 690         struct kobject *kobj, struct attribute *attr, char *buf)
 691 {
 692         int i;
 693         ssize_t ret = 0;
 694         unsigned long flags;
 695
 696         spin_lock_irqsave(&target_loads_lock, flags);
 697
 698         for (i = 0; i < ntarget_loads; i++)
 699                 ret += sprintf(buf + ret, "%u%s", target_loads[i],
 700                                i & 0x1 ? ":" : " ");
 701
 702         ret += sprintf(buf + ret, "\n");
 703         spin_unlock_irqrestore(&target_loads_lock, flags);
 704         return ret;
 705 }
 706
 707 static ssize_t store_target_loads(
 708         struct kobject *kobj, struct attribute *attr, const char *buf,
 709         size_t count)
 710 {
 711         int ntokens;
 712         unsigned int *new_target_loads = NULL;
 713         unsigned long flags;
 714
 715         new_target_loads = get_tokenized_data(buf, &ntokens);
 716         if (IS_ERR(new_target_loads))
 717                 return PTR_RET(new_target_loads);
 718
 719         spin_lock_irqsave(&target_loads_lock, flags);
 720         if (target_loads != default_target_loads)
 721                 kfree(target_loads);
 722         target_loads = new_target_loads;
 723         ntarget_loads = ntokens;
 724         spin_unlock_irqrestore(&target_loads_lock, flags);
 725         return count;
 726 }
 727
 728 static struct global_attr target_loads_attr =
 729         __ATTR(target_loads, S_IRUGO | S_IWUSR,
 730                 show_target_loads, store_target_loads);
 731
 732 static ssize_t show_above_hispeed_delay(
 733         struct kobject *kobj, struct attribute *attr, char *buf)
 734 {
 735         int i;
 736         ssize_t ret = 0;
 737         unsigned long flags;
 738
 739         spin_lock_irqsave(&above_hispeed_delay_lock, flags);
 740
 741         for (i = 0; i < nabove_hispeed_delay; i++)
 742                 ret += sprintf(buf + ret, "%u%s", above_hispeed_delay[i],
 743                                i & 0x1 ? ":" : " ");
 744
 745         ret += sprintf(buf + ret, "\n");
 746         spin_unlock_irqrestore(&above_hispeed_delay_lock, flags);
 747         return ret;
 748 }
 749
 750 static ssize_t store_above_hispeed_delay(
 751         struct kobject *kobj, struct attribute *attr, const char *buf,
 752         size_t count)
 753 {
 754         int ntokens;
 755         unsigned int *new_above_hispeed_delay = NULL;
 756         unsigned long flags;
 757
 758         new_above_hispeed_delay = get_tokenized_data(buf, &ntokens);
 759         if (IS_ERR(new_above_hispeed_delay))
 760                 return PTR_RET(new_above_hispeed_delay);
 761
 762         spin_lock_irqsave(&above_hispeed_delay_lock, flags);
 763         if (above_hispeed_delay != default_above_hispeed_delay)
 764                 kfree(above_hispeed_delay);
 765         above_hispeed_delay = new_above_hispeed_delay;
 766         nabove_hispeed_delay = ntokens;
 767         spin_unlock_irqrestore(&above_hispeed_delay_lock, flags);
 768         return count;
 769
 770 }
 771
 772 static struct global_attr above_hispeed_delay_attr =
 773         __ATTR(above_hispeed_delay, S_IRUGO | S_IWUSR,
 774                 show_above_hispeed_delay, store_above_hispeed_delay);
 775
 776 static ssize_t show_hispeed_freq(struct kobject *kobj,
 777                                  struct attribute *attr, char *buf)
 778 {
 779         return sprintf(buf, "%u\n", hispeed_freq);
 780 }
 781
 782 static ssize_t store_hispeed_freq(struct kobject *kobj,
 783                                   struct attribute *attr, const char *buf,
 784                                   size_t count)
 785 {
 786         int ret;
 787         long unsigned int val;
 788
 789         ret = strict_strtoul(buf, 0, &val);
 790         if (ret < 0)
 791                 return ret;
 792         hispeed_freq = val;
 793         return count;
 794 }
 795
 796 static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644,
 797                 show_hispeed_freq, store_hispeed_freq);
 798
 799
 800 static ssize_t show_go_hispeed_load(struct kobject *kobj,
 801                                      struct attribute *attr, char *buf)
 802 {
 803         return sprintf(buf, "%lu\n", go_hispeed_load);
 804 }
 805
 806 static ssize_t store_go_hispeed_load(struct kobject *kobj,
 807                         struct attribute *attr, const char *buf, size_t count)
 808 {
 809         int ret;
 810         unsigned long val;
 811
 812         ret = strict_strtoul(buf, 0, &val);
 813         if (ret < 0)
 814                 return ret;
 815         go_hispeed_load = val;
 816         return count;
 817 }
 818
 819 static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644,
 820                 show_go_hispeed_load, store_go_hispeed_load);
 821
 822 static ssize_t show_min_sample_time(struct kobject *kobj,
 823                                 struct attribute *attr, char *buf)
 824 {
 825         return sprintf(buf, "%lu\n", min_sample_time);
 826 }
 827
 828 static ssize_t store_min_sample_time(struct kobject *kobj,
 829                         struct attribute *attr, const char *buf, size_t count)
 830 {
 831         int ret;
 832         unsigned long val;
 833
 834         ret = strict_strtoul(buf, 0, &val);
 835         if (ret < 0)
 836                 return ret;
 837         min_sample_time = val;
 838         return count;
 839 }
 840
 841 static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
 842                 show_min_sample_time, store_min_sample_time);
 843
 844 static ssize_t show_timer_rate(struct kobject *kobj,
 845                         struct attribute *attr, char *buf)
 846 {
 847         return sprintf(buf, "%lu\n", timer_rate);
 848 }
 849
 850 static ssize_t store_timer_rate(struct kobject *kobj,
 851                         struct attribute *attr, const char *buf, size_t count)
 852 {
 853         int ret;
 854         unsigned long val;
 855
 856         ret = strict_strtoul(buf, 0, &val);
 857         if (ret < 0)
 858                 return ret;
 859         timer_rate = val;
 860         return count;
 861 }
 862
 863 static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
 864                 show_timer_rate, store_timer_rate);
 865
 866 static ssize_t show_timer_slack(
 867         struct kobject *kobj, struct attribute *attr, char *buf)
 868 {
 869         return sprintf(buf, "%d\n", timer_slack_val);
 870 }
 871
 872 static ssize_t store_timer_slack(
 873         struct kobject *kobj, struct attribute *attr, const char *buf,
 874         size_t count)
 875 {
 876         int ret;
 877         unsigned long val;
 878
 879         ret = kstrtol(buf, 10, &val);
 880         if (ret < 0)
 881                 return ret;
 882
 883         timer_slack_val = val;
 884         return count;
 885 }
 886
 887 define_one_global_rw(timer_slack);
 888
 889 static ssize_t show_boost(struct kobject *kobj, struct attribute *attr,
 890                           char *buf)
 891 {
 892         return sprintf(buf, "%d\n", boost_val);
 893 }
 894
 895 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 896                            const char *buf, size_t count)
 897 {
 898         int ret;
 899         unsigned long val;
 900
 901         ret = kstrtoul(buf, 0, &val);
 902         if (ret < 0)
 903                 return ret;
 904
 905         boost_val = val;
 906
 907         if (boost_val) {
 908                 trace_cpufreq_interactive_boost("on");
 909                 cpufreq_interactive_boost();
 910         } else {
 911                 trace_cpufreq_interactive_unboost("off");
 912         }
 913
 914         return count;
 915 }
 916
 917 define_one_global_rw(boost);
 918
 919 static ssize_t store_boostpulse(struct kobject *kobj, struct attribute *attr,
 920                                 const char *buf, size_t count)
 921 {
 922         int ret;
 923         unsigned long val;
 924
 925         ret = kstrtoul(buf, 0, &val);
 926         if (ret < 0)
 927                 return ret;
 928
 929         boostpulse_endtime = ktime_to_us(ktime_get()) + boostpulse_duration_val;
 930         trace_cpufreq_interactive_boost("pulse");
 931         cpufreq_interactive_boost();
 932         return count;
 933 }
 934
 935 static struct global_attr boostpulse =
 936         __ATTR(boostpulse, 0200, NULL, store_boostpulse);
 937
 938 static ssize_t show_boostpulse_duration(
 939         struct kobject *kobj, struct attribute *attr, char *buf)
 940 {
 941         return sprintf(buf, "%d\n", boostpulse_duration_val);
 942 }
 943
 944 static ssize_t store_boostpulse_duration(
 945         struct kobject *kobj, struct attribute *attr, const char *buf,
 946         size_t count)
 947 {
 948         int ret;
 949         unsigned long val;
 950
 951         ret = kstrtoul(buf, 0, &val);
 952         if (ret < 0)
 953                 return ret;
 954
 955         boostpulse_duration_val = val;
 956         return count;
 957 }
 958
 959 define_one_global_rw(boostpulse_duration);
 960
 961 static ssize_t show_io_is_busy(struct kobject *kobj,
 962                         struct attribute *attr, char *buf)
 963 {
 964         return sprintf(buf, "%u\n", io_is_busy);
 965 }
 966
 967 static ssize_t store_io_is_busy(struct kobject *kobj,
 968                         struct attribute *attr, const char *buf, size_t count)
 969 {
 970         int ret;
 971         unsigned long val;
 972
 973         ret = kstrtoul(buf, 0, &val);
 974         if (ret < 0)
 975                 return ret;
 976         io_is_busy = val;
 977         return count;
 978 }
 979
 980 static struct global_attr io_is_busy_attr = __ATTR(io_is_busy, 0644,
 981                 show_io_is_busy, store_io_is_busy);
 982
 983 static struct attribute *interactive_attributes[] = {
 984         &target_loads_attr.attr,
 985         &above_hispeed_delay_attr.attr,
 986         &hispeed_freq_attr.attr,
 987         &go_hispeed_load_attr.attr,
 988         &min_sample_time_attr.attr,
 989         &timer_rate_attr.attr,
 990         &timer_slack.attr,
 991         &boost.attr,
 992         &boostpulse.attr,
 993         &boostpulse_duration.attr,
 994         &io_is_busy_attr.attr,
 995         NULL,
 996 };
 997
 998 static struct attribute_group interactive_attr_group = {
 999         .attrs = interactive_attributes,
1000         .name = "interactive",
1001 };
1002
1003 static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
1004                                              unsigned long val,
1005                                              void *data)
1006 {
1007         switch (val) {
1008         case IDLE_START:
1009                 cpufreq_interactive_idle_start();
1010                 break;
1011         case IDLE_END:
1012                 cpufreq_interactive_idle_end();
1013                 break;
1014         }
1015
1016         return 0;
1017 }
1018
1019 static struct notifier_block cpufreq_interactive_idle_nb = {
1020         .notifier_call = cpufreq_interactive_idle_notifier,
1021 };
1022
1023 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
1024                 unsigned int event)
1025 {
1026         int rc;
1027         unsigned int j;
1028         struct cpufreq_interactive_cpuinfo *pcpu;
1029         struct cpufreq_frequency_table *freq_table;
1030
1031         switch (event) {
1032         case CPUFREQ_GOV_START:
1033                 if (!cpu_online(policy->cpu))
1034                         return -EINVAL;
1035
1036                 mutex_lock(&gov_lock);
1037
1038                 freq_table =
1039                         cpufreq_frequency_get_table(policy->cpu);
1040                 if (!hispeed_freq)
1041                         hispeed_freq = policy->max;
1042
1043                 for_each_cpu(j, policy->cpus) {
1044                         unsigned long expires;
1045
1046                         pcpu = &per_cpu(cpuinfo, j);
1047                         pcpu->policy = policy;
1048                         pcpu->target_freq = policy->cur;
1049                         pcpu->freq_table = freq_table;
1050                         pcpu->floor_freq = pcpu->target_freq;
1051                         pcpu->floor_validate_time =
1052                                 ktime_to_us(ktime_get());
1053                         pcpu->hispeed_validate_time =
1054                                 pcpu->floor_validate_time;
1055                         down_write(&pcpu->enable_sem);
1056                         expires = jiffies + usecs_to_jiffies(timer_rate);
1057                         pcpu->cpu_timer.expires = expires;
1058                         add_timer_on(&pcpu->cpu_timer, j);
1059                         if (timer_slack_val >= 0) {
1060                                 expires += usecs_to_jiffies(timer_slack_val);
1061                                 pcpu->cpu_slack_timer.expires = expires;
1062                                 add_timer_on(&pcpu->cpu_slack_timer, j);
1063                         }
1064                         pcpu->governor_enabled = 1;
1065                         up_write(&pcpu->enable_sem);
1066                 }
1067
1068                 /*
1069                  * Do not register the idle hook and create sysfs
1070                  * entries if we have already done so.
1071                  */
1072                 if (++active_count > 1) {
1073                         mutex_unlock(&gov_lock);
1074                         return 0;
1075                 }
1076
1077                 rc = sysfs_create_group(cpufreq_global_kobject,
1078                                 &interactive_attr_group);
1079                 if (rc) {
1080                         mutex_unlock(&gov_lock);
1081                         return rc;
1082                 }
1083
1084                 idle_notifier_register(&cpufreq_interactive_idle_nb);
1085                 cpufreq_register_notifier(
1086                         &cpufreq_notifier_block, CPUFREQ_TRANSITION_NOTIFIER);
1087                 mutex_unlock(&gov_lock);
1088                 break;
1089
1090         case CPUFREQ_GOV_STOP:
1091                 mutex_lock(&gov_lock);
1092                 for_each_cpu(j, policy->cpus) {
1093                         pcpu = &per_cpu(cpuinfo, j);
1094                         down_write(&pcpu->enable_sem);
1095                         pcpu->governor_enabled = 0;
1096                         del_timer_sync(&pcpu->cpu_timer);
1097                         del_timer_sync(&pcpu->cpu_slack_timer);
1098                         up_write(&pcpu->enable_sem);
1099                 }
1100
1101                 if (--active_count > 0) {
1102                         mutex_unlock(&gov_lock);
1103                         return 0;
1104                 }
1105
1106                 cpufreq_unregister_notifier(
1107                         &cpufreq_notifier_block, CPUFREQ_TRANSITION_NOTIFIER);
1108                 idle_notifier_unregister(&cpufreq_interactive_idle_nb);
1109                 sysfs_remove_group(cpufreq_global_kobject,
1110                                 &interactive_attr_group);
1111                 mutex_unlock(&gov_lock);
1112
1113                 break;
1114
1115         case CPUFREQ_GOV_LIMITS:
1116                 if (policy->max < policy->cur)
1117                         __cpufreq_driver_target(policy,
1118                                         policy->max, CPUFREQ_RELATION_H);
1119                 else if (policy->min > policy->cur)
1120                         __cpufreq_driver_target(policy,
1121                                         policy->min, CPUFREQ_RELATION_L);
1122                 break;
1123         }
1124         return 0;
1125 }
1126
1127 static void cpufreq_interactive_nop_timer(unsigned long data)
1128 {
1129 }
1130
1131 static int __init cpufreq_interactive_init(void)
1132 {
1133         unsigned int i;
1134         struct cpufreq_interactive_cpuinfo *pcpu;
1135         struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
1136
1137         /* Initalize per-cpu timers */
1138         for_each_possible_cpu(i) {
1139                 pcpu = &per_cpu(cpuinfo, i);
1140                 init_timer_deferrable(&pcpu->cpu_timer);
1141                 pcpu->cpu_timer.function = cpufreq_interactive_timer;
1142                 pcpu->cpu_timer.data = i;
1143                 init_timer(&pcpu->cpu_slack_timer);
1144                 pcpu->cpu_slack_timer.function = cpufreq_interactive_nop_timer;
1145                 spin_lock_init(&pcpu->load_lock);
1146                 init_rwsem(&pcpu->enable_sem);
1147         }
1148
1149         spin_lock_init(&target_loads_lock);
1150         spin_lock_init(&speedchange_cpumask_lock);
1151         mutex_init(&gov_lock);
1152         speedchange_task =
1153                 kthread_create(cpufreq_interactive_speedchange_task, NULL,
1154                                "cfinteractive");
1155         if (IS_ERR(speedchange_task))
1156                 return PTR_ERR(speedchange_task);
1157
1158         sched_setscheduler_nocheck(speedchange_task, SCHED_FIFO, &param);
1159         get_task_struct(speedchange_task);
1160
1161         /* NB: wake up so the thread does not look hung to the freezer */
1162         wake_up_process(speedchange_task);
1163
1164         return cpufreq_register_governor(&cpufreq_gov_interactive);
1165 }
1166
1167 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
1168 fs_initcall(cpufreq_interactive_init);
1169 #else
1170 module_init(cpufreq_interactive_init);
1171 #endif
1172
1173 static void __exit cpufreq_interactive_exit(void)
1174 {
1175         cpufreq_unregister_governor(&cpufreq_gov_interactive);
1176         kthread_stop(speedchange_task);
1177         put_task_struct(speedchange_task);
1178 }
1179
1180 module_exit(cpufreq_interactive_exit);
1181
1182 MODULE_AUTHOR("Mike Chan <mike@android.com>");
1183 MODULE_DESCRIPTION("'cpufreq_interactive' - A cpufreq governor for "
1184         "Latency sensitive workloads");
1185 MODULE_LICENSE("GPL");