drivers/cpufreq/cpufreq_interactive.c

   1 /*
   2  * drivers/cpufreq/cpufreq_interactive.c
   3  *
   4  * Copyright (C) 2010 Google, Inc.
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * Author: Mike Chan (mike@android.com)
  16  *
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/cpumask.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/module.h>
  23 #include <linux/moduleparam.h>
  24 #include <linux/rwsem.h>
  25 #include <linux/sched.h>
  26 #include <linux/sched/rt.h>
  27 #include <linux/tick.h>
  28 #include <linux/time.h>
  29 #include <linux/timer.h>
  30 #include <linux/workqueue.h>
  31 #include <linux/kthread.h>
  32 #include <linux/slab.h>
  33 #include <linux/kernel_stat.h>
  34 #include <asm/cputime.h>
  35
  36 #define CREATE_TRACE_POINTS
  37 #include <trace/events/cpufreq_interactive.h>
  38
  39 static int active_count;
  40
  41 struct cpufreq_interactive_cpuinfo {
  42         struct timer_list cpu_timer;
  43         struct timer_list cpu_slack_timer;
  44         spinlock_t load_lock; /* protects the next 4 fields */
  45         u64 time_in_idle;
  46         u64 time_in_idle_timestamp;
  47         u64 cputime_speedadj;
  48         u64 cputime_speedadj_timestamp;
  49         struct cpufreq_policy *policy;
  50         struct cpufreq_frequency_table *freq_table;
  51         unsigned int target_freq;
  52         unsigned int floor_freq;
  53         u64 floor_validate_time;
  54         u64 hispeed_validate_time;
  55         struct rw_semaphore enable_sem;
  56         int governor_enabled;
  57 };
  58
  59 static DEFINE_PER_CPU(struct cpufreq_interactive_cpuinfo, cpuinfo);
  60
  61 /* realtime thread handles frequency scaling */
  62 static struct task_struct *speedchange_task;
  63 static cpumask_t speedchange_cpumask;
  64 static spinlock_t speedchange_cpumask_lock;
  65 static struct mutex gov_lock;
  66
  67 /* Hi speed to bump to from lo speed when load burst (default max) */
  68 static unsigned int hispeed_freq;
  69
  70 /* Go to hi speed when CPU load at or above this value. */
  71 #define DEFAULT_GO_HISPEED_LOAD 99
  72 static unsigned long go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
  73
  74 /* Target load.  Lower values result in higher CPU speeds. */
  75 #define DEFAULT_TARGET_LOAD 90
  76 static unsigned int default_target_loads[] = {DEFAULT_TARGET_LOAD};
  77 static spinlock_t target_loads_lock;
  78 static unsigned int *target_loads = default_target_loads;
  79 static int ntarget_loads = ARRAY_SIZE(default_target_loads);
  80
  81 /*
  82  * The minimum amount of time to spend at a frequency before we can ramp down.
  83  */
  84 #define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
  85 static unsigned long min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
  86
  87 /*
  88  * The sample rate of the timer used to increase frequency
  89  */
  90 #define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC)
  91 static unsigned long timer_rate = DEFAULT_TIMER_RATE;
  92
  93 /*
  94  * Wait this long before raising speed above hispeed, by default a single
  95  * timer interval.
  96  */
  97 #define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE
  98 static unsigned int default_above_hispeed_delay[] = {
  99         DEFAULT_ABOVE_HISPEED_DELAY };
 100 static spinlock_t above_hispeed_delay_lock;
 101 static unsigned int *above_hispeed_delay = default_above_hispeed_delay;
 102 static int nabove_hispeed_delay = ARRAY_SIZE(default_above_hispeed_delay);
 103
 104 /* Non-zero means indefinite speed boost active */
 105 static int boost_val;
 106 /* Duration of a boot pulse in usecs */
 107 static int boostpulse_duration_val = DEFAULT_MIN_SAMPLE_TIME;
 108 /* End time of boost pulse in ktime converted to usecs */
 109 static u64 boostpulse_endtime;
 110
 111 /*
 112  * Max additional time to wait in idle, beyond timer_rate, at speeds above
 113  * minimum before wakeup to reduce speed, or -1 if unnecessary.
 114  */
 115 #define DEFAULT_TIMER_SLACK (4 * DEFAULT_TIMER_RATE)
 116 static int timer_slack_val = DEFAULT_TIMER_SLACK;
 117
 118 static bool io_is_busy;
 119
 120 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 121                 unsigned int event);
 122
 123 #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 124 static
 125 #endif
 126 struct cpufreq_governor cpufreq_gov_interactive = {
 127         .name = "interactive",
 128         .governor = cpufreq_governor_interactive,
 129         .max_transition_latency = 10000000,
 130         .owner = THIS_MODULE,
 131 };
 132
 133 static inline cputime64_t get_cpu_idle_time_jiffy(unsigned int cpu,
 134                                                   cputime64_t *wall)
 135 {
 136         u64 idle_time;
 137         u64 cur_wall_time;
 138         u64 busy_time;
 139
 140         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 141
 142         busy_time  = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 143         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 144         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 145         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 146         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 147         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 148
 149         idle_time = cur_wall_time - busy_time;
 150         if (wall)
 151                 *wall = jiffies_to_usecs(cur_wall_time);
 152
 153         return jiffies_to_usecs(idle_time);
 154 }
 155
 156 static inline cputime64_t get_cpu_idle_time(unsigned int cpu,
 157                                             cputime64_t *wall)
 158 {
 159         u64 idle_time = get_cpu_idle_time_us(cpu, wall);
 160
 161         if (idle_time == -1ULL)
 162                 idle_time = get_cpu_idle_time_jiffy(cpu, wall);
 163         else if (!io_is_busy)
 164                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 165
 166         return idle_time;
 167 }
 168
 169 static void cpufreq_interactive_timer_resched(
 170         struct cpufreq_interactive_cpuinfo *pcpu)
 171 {
 172         unsigned long expires;
 173         unsigned long flags;
 174
 175         spin_lock_irqsave(&pcpu->load_lock, flags);
 176         pcpu->time_in_idle =
 177                 get_cpu_idle_time(smp_processor_id(),
 178                                      &pcpu->time_in_idle_timestamp);
 179         pcpu->cputime_speedadj = 0;
 180         pcpu->cputime_speedadj_timestamp = pcpu->time_in_idle_timestamp;
 181         expires = jiffies + usecs_to_jiffies(timer_rate);
 182         mod_timer_pinned(&pcpu->cpu_timer, expires);
 183
 184         if (timer_slack_val >= 0 && pcpu->target_freq > pcpu->policy->min) {
 185                 expires += usecs_to_jiffies(timer_slack_val);
 186                 mod_timer_pinned(&pcpu->cpu_slack_timer, expires);
 187         }
 188
 189         spin_unlock_irqrestore(&pcpu->load_lock, flags);
 190 }
 191
 192 static unsigned int freq_to_above_hispeed_delay(unsigned int freq)
 193 {
 194         int i;
 195         unsigned int ret;
 196         unsigned long flags;
 197
 198         spin_lock_irqsave(&above_hispeed_delay_lock, flags);
 199
 200         for (i = 0; i < nabove_hispeed_delay - 1 &&
 201                         freq >= above_hispeed_delay[i+1]; i += 2)
 202                 ;
 203
 204         ret = above_hispeed_delay[i];
 205         spin_unlock_irqrestore(&above_hispeed_delay_lock, flags);
 206         return ret;
 207 }
 208
 209 static unsigned int freq_to_targetload(unsigned int freq)
 210 {
 211         int i;
 212         unsigned int ret;
 213         unsigned long flags;
 214
 215         spin_lock_irqsave(&target_loads_lock, flags);
 216
 217         for (i = 0; i < ntarget_loads - 1 && freq >= target_loads[i+1]; i += 2)
 218                 ;
 219
 220         ret = target_loads[i];
 221         spin_unlock_irqrestore(&target_loads_lock, flags);
 222         return ret;
 223 }
 224
 225 /*
 226  * If increasing frequencies never map to a lower target load then
 227  * choose_freq() will find the minimum frequency that does not exceed its
 228  * target load given the current load.
 229  */
 230
 231 static unsigned int choose_freq(
 232         struct cpufreq_interactive_cpuinfo *pcpu, unsigned int loadadjfreq)
 233 {
 234         unsigned int freq = pcpu->policy->cur;
 235         unsigned int prevfreq, freqmin, freqmax;
 236         unsigned int tl;
 237         int index;
 238
 239         freqmin = 0;
 240         freqmax = UINT_MAX;
 241
 242         do {
 243                 prevfreq = freq;
 244                 tl = freq_to_targetload(freq);
 245
 246                 /*
 247                  * Find the lowest frequency where the computed load is less
 248                  * than or equal to the target load.
 249                  */
 250
 251                 if (cpufreq_frequency_table_target(
 252                             pcpu->policy, pcpu->freq_table, loadadjfreq / tl,
 253                             CPUFREQ_RELATION_L, &index))
 254                         break;
 255                 freq = pcpu->freq_table[index].frequency;
 256
 257                 if (freq > prevfreq) {
 258                         /* The previous frequency is too low. */
 259                         freqmin = prevfreq;
 260
 261                         if (freq >= freqmax) {
 262                                 /*
 263                                  * Find the highest frequency that is less
 264                                  * than freqmax.
 265                                  */
 266                                 if (cpufreq_frequency_table_target(
 267                                             pcpu->policy, pcpu->freq_table,
 268                                             freqmax - 1, CPUFREQ_RELATION_H,
 269                                             &index))
 270                                         break;
 271                                 freq = pcpu->freq_table[index].frequency;
 272
 273                                 if (freq == freqmin) {
 274                                         /*
 275                                          * The first frequency below freqmax
 276                                          * has already been found to be too
 277                                          * low.  freqmax is the lowest speed
 278                                          * we found that is fast enough.
 279                                          */
 280                                         freq = freqmax;
 281                                         break;
 282                                 }
 283                         }
 284                 } else if (freq < prevfreq) {
 285                         /* The previous frequency is high enough. */
 286                         freqmax = prevfreq;
 287
 288                         if (freq <= freqmin) {
 289                                 /*
 290                                  * Find the lowest frequency that is higher
 291                                  * than freqmin.
 292                                  */
 293                                 if (cpufreq_frequency_table_target(
 294                                             pcpu->policy, pcpu->freq_table,
 295                                             freqmin + 1, CPUFREQ_RELATION_L,
 296                                             &index))
 297                                         break;
 298                                 freq = pcpu->freq_table[index].frequency;
 299
 300                                 /*
 301                                  * If freqmax is the first frequency above
 302                                  * freqmin then we have already found that
 303                                  * this speed is fast enough.
 304                                  */
 305                                 if (freq == freqmax)
 306                                         break;
 307                         }
 308                 }
 309
 310                 /* If same frequency chosen as previous then done. */
 311         } while (freq != prevfreq);
 312
 313         return freq;
 314 }
 315
 316 static u64 update_load(int cpu)
 317 {
 318         struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, cpu);
 319         u64 now;
 320         u64 now_idle;
 321         unsigned int delta_idle;
 322         unsigned int delta_time;
 323         u64 active_time;
 324
 325         now_idle = get_cpu_idle_time(cpu, &now);
 326         delta_idle = (unsigned int)(now_idle - pcpu->time_in_idle);
 327         delta_time = (unsigned int)(now - pcpu->time_in_idle_timestamp);
 328
 329         if (delta_time <= delta_idle)
 330                 active_time = 0;
 331         else
 332                 active_time = delta_time - delta_idle;
 333
 334         pcpu->cputime_speedadj += active_time * pcpu->policy->cur;
 335
 336         pcpu->time_in_idle = now_idle;
 337         pcpu->time_in_idle_timestamp = now;
 338         return now;
 339 }
 340
 341 static void cpufreq_interactive_timer(unsigned long data)
 342 {
 343         u64 now;
 344         unsigned int delta_time;
 345         u64 cputime_speedadj;
 346         int cpu_load;
 347         struct cpufreq_interactive_cpuinfo *pcpu =
 348                 &per_cpu(cpuinfo, data);
 349         unsigned int new_freq;
 350         unsigned int loadadjfreq;
 351         unsigned int index;
 352         unsigned long flags;
 353         bool boosted;
 354
 355         if (!down_read_trylock(&pcpu->enable_sem))
 356                 return;
 357         if (!pcpu->governor_enabled)
 358                 goto exit;
 359
 360         spin_lock_irqsave(&pcpu->load_lock, flags);
 361         now = update_load(data);
 362         delta_time = (unsigned int)(now - pcpu->cputime_speedadj_timestamp);
 363         cputime_speedadj = pcpu->cputime_speedadj;
 364         spin_unlock_irqrestore(&pcpu->load_lock, flags);
 365
 366         if (WARN_ON_ONCE(!delta_time))
 367                 goto rearm;
 368
 369         do_div(cputime_speedadj, delta_time);
 370         loadadjfreq = (unsigned int)cputime_speedadj * 100;
 371         cpu_load = loadadjfreq / pcpu->target_freq;
 372         boosted = boost_val || now < boostpulse_endtime;
 373
 374         if (cpu_load >= go_hispeed_load || boosted) {
 375                 if (pcpu->target_freq < hispeed_freq) {
 376                         new_freq = hispeed_freq;
 377                 } else {
 378                         new_freq = choose_freq(pcpu, loadadjfreq);
 379
 380                         if (new_freq < hispeed_freq)
 381                                 new_freq = hispeed_freq;
 382                 }
 383         } else {
 384                 new_freq = choose_freq(pcpu, loadadjfreq);
 385         }
 386
 387         if (pcpu->target_freq >= hispeed_freq &&
 388             new_freq > pcpu->target_freq &&
 389             now - pcpu->hispeed_validate_time <
 390             freq_to_above_hispeed_delay(pcpu->target_freq)) {
 391                 trace_cpufreq_interactive_notyet(
 392                         data, cpu_load, pcpu->target_freq,
 393                         pcpu->policy->cur, new_freq);
 394                 goto rearm;
 395         }
 396
 397         pcpu->hispeed_validate_time = now;
 398
 399         if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
 400                                            new_freq, CPUFREQ_RELATION_L,
 401                                            &index))
 402                 goto rearm;
 403
 404         new_freq = pcpu->freq_table[index].frequency;
 405
 406         /*
 407          * Do not scale below floor_freq unless we have been at or above the
 408          * floor frequency for the minimum sample time since last validated.
 409          */
 410         if (new_freq < pcpu->floor_freq) {
 411                 if (now - pcpu->floor_validate_time < min_sample_time) {
 412                         trace_cpufreq_interactive_notyet(
 413                                 data, cpu_load, pcpu->target_freq,
 414                                 pcpu->policy->cur, new_freq);
 415                         goto rearm;
 416                 }
 417         }
 418
 419         /*
 420          * Update the timestamp for checking whether speed has been held at
 421          * or above the selected frequency for a minimum of min_sample_time,
 422          * if not boosted to hispeed_freq.  If boosted to hispeed_freq then we
 423          * allow the speed to drop as soon as the boostpulse duration expires
 424          * (or the indefinite boost is turned off).
 425          */
 426
 427         if (!boosted || new_freq > hispeed_freq) {
 428                 pcpu->floor_freq = new_freq;
 429                 pcpu->floor_validate_time = now;
 430         }
 431
 432         if (pcpu->target_freq == new_freq) {
 433                 trace_cpufreq_interactive_already(
 434                         data, cpu_load, pcpu->target_freq,
 435                         pcpu->policy->cur, new_freq);
 436                 goto rearm_if_notmax;
 437         }
 438
 439         trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
 440                                          pcpu->policy->cur, new_freq);
 441
 442         pcpu->target_freq = new_freq;
 443         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 444         cpumask_set_cpu(data, &speedchange_cpumask);
 445         spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 446         wake_up_process(speedchange_task);
 447
 448 rearm_if_notmax:
 449         /*
 450          * Already set max speed and don't see a need to change that,
 451          * wait until next idle to re-evaluate, don't need timer.
 452          */
 453         if (pcpu->target_freq == pcpu->policy->max)
 454                 goto exit;
 455
 456 rearm:
 457         if (!timer_pending(&pcpu->cpu_timer))
 458                 cpufreq_interactive_timer_resched(pcpu);
 459
 460 exit:
 461         up_read(&pcpu->enable_sem);
 462         return;
 463 }
 464
 465 static void cpufreq_interactive_idle_start(void)
 466 {
 467         struct cpufreq_interactive_cpuinfo *pcpu =
 468                 &per_cpu(cpuinfo, smp_processor_id());
 469         int pending;
 470
 471         if (!down_read_trylock(&pcpu->enable_sem))
 472                 return;
 473         if (!pcpu->governor_enabled) {
 474                 up_read(&pcpu->enable_sem);
 475                 return;
 476         }
 477
 478         pending = timer_pending(&pcpu->cpu_timer);
 479
 480         if (pcpu->target_freq != pcpu->policy->min) {
 481                 /*
 482                  * Entering idle while not at lowest speed.  On some
 483                  * platforms this can hold the other CPU(s) at that speed
 484                  * even though the CPU is idle. Set a timer to re-evaluate
 485                  * speed so this idle CPU doesn't hold the other CPUs above
 486                  * min indefinitely.  This should probably be a quirk of
 487                  * the CPUFreq driver.
 488                  */
 489                 if (!pending)
 490                         cpufreq_interactive_timer_resched(pcpu);
 491         }
 492
 493         up_read(&pcpu->enable_sem);
 494 }
 495
 496 static void cpufreq_interactive_idle_end(void)
 497 {
 498         struct cpufreq_interactive_cpuinfo *pcpu =
 499                 &per_cpu(cpuinfo, smp_processor_id());
 500
 501         if (!down_read_trylock(&pcpu->enable_sem))
 502                 return;
 503         if (!pcpu->governor_enabled) {
 504                 up_read(&pcpu->enable_sem);
 505                 return;
 506         }
 507
 508         /* Arm the timer for 1-2 ticks later if not already. */
 509         if (!timer_pending(&pcpu->cpu_timer)) {
 510                 cpufreq_interactive_timer_resched(pcpu);
 511         } else if (time_after_eq(jiffies, pcpu->cpu_timer.expires)) {
 512                 del_timer(&pcpu->cpu_timer);
 513                 del_timer(&pcpu->cpu_slack_timer);
 514                 cpufreq_interactive_timer(smp_processor_id());
 515         }
 516
 517         up_read(&pcpu->enable_sem);
 518 }
 519
 520 static int cpufreq_interactive_speedchange_task(void *data)
 521 {
 522         unsigned int cpu;
 523         cpumask_t tmp_mask;
 524         unsigned long flags;
 525         struct cpufreq_interactive_cpuinfo *pcpu;
 526
 527         while (1) {
 528                 set_current_state(TASK_INTERRUPTIBLE);
 529                 spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 530
 531                 if (cpumask_empty(&speedchange_cpumask)) {
 532                         spin_unlock_irqrestore(&speedchange_cpumask_lock,
 533                                                flags);
 534                         schedule();
 535
 536                         if (kthread_should_stop())
 537                                 break;
 538
 539                         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 540                 }
 541
 542                 set_current_state(TASK_RUNNING);
 543                 tmp_mask = speedchange_cpumask;
 544                 cpumask_clear(&speedchange_cpumask);
 545                 spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 546
 547                 for_each_cpu(cpu, &tmp_mask) {
 548                         unsigned int j;
 549                         unsigned int max_freq = 0;
 550
 551                         pcpu = &per_cpu(cpuinfo, cpu);
 552                         if (!down_read_trylock(&pcpu->enable_sem))
 553                                 continue;
 554                         if (!pcpu->governor_enabled) {
 555                                 up_read(&pcpu->enable_sem);
 556                                 continue;
 557                         }
 558
 559                         for_each_cpu(j, pcpu->policy->cpus) {
 560                                 struct cpufreq_interactive_cpuinfo *pjcpu =
 561                                         &per_cpu(cpuinfo, j);
 562
 563                                 if (pjcpu->target_freq > max_freq)
 564                                         max_freq = pjcpu->target_freq;
 565                         }
 566
 567                         if (max_freq != pcpu->policy->cur)
 568                                 __cpufreq_driver_target(pcpu->policy,
 569                                                         max_freq,
 570                                                         CPUFREQ_RELATION_H);
 571                         trace_cpufreq_interactive_setspeed(cpu,
 572                                                      pcpu->target_freq,
 573                                                      pcpu->policy->cur);
 574
 575                         up_read(&pcpu->enable_sem);
 576                 }
 577         }
 578
 579         return 0;
 580 }
 581
 582 static void cpufreq_interactive_boost(void)
 583 {
 584         int i;
 585         int anyboost = 0;
 586         unsigned long flags;
 587         struct cpufreq_interactive_cpuinfo *pcpu;
 588
 589         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 590
 591         for_each_online_cpu(i) {
 592                 pcpu = &per_cpu(cpuinfo, i);
 593
 594                 if (pcpu->target_freq < hispeed_freq) {
 595                         pcpu->target_freq = hispeed_freq;
 596                         cpumask_set_cpu(i, &speedchange_cpumask);
 597                         pcpu->hispeed_validate_time =
 598                                 ktime_to_us(ktime_get());
 599                         anyboost = 1;
 600                 }
 601
 602                 /*
 603                  * Set floor freq and (re)start timer for when last
 604                  * validated.
 605                  */
 606
 607                 pcpu->floor_freq = hispeed_freq;
 608                 pcpu->floor_validate_time = ktime_to_us(ktime_get());
 609         }
 610
 611         spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 612
 613         if (anyboost)
 614                 wake_up_process(speedchange_task);
 615 }
 616
 617 static int cpufreq_interactive_notifier(
 618         struct notifier_block *nb, unsigned long val, void *data)
 619 {
 620         struct cpufreq_freqs *freq = data;
 621         struct cpufreq_interactive_cpuinfo *pcpu;
 622         int cpu;
 623         unsigned long flags;
 624
 625         if (val == CPUFREQ_POSTCHANGE) {
 626                 pcpu = &per_cpu(cpuinfo, freq->cpu);
 627                 if (!down_read_trylock(&pcpu->enable_sem))
 628                         return 0;
 629                 if (!pcpu->governor_enabled) {
 630                         up_read(&pcpu->enable_sem);
 631                         return 0;
 632                 }
 633
 634                 for_each_cpu(cpu, pcpu->policy->cpus) {
 635                         struct cpufreq_interactive_cpuinfo *pjcpu =
 636                                 &per_cpu(cpuinfo, cpu);
 637                         spin_lock_irqsave(&pjcpu->load_lock, flags);
 638                         update_load(cpu);
 639                         spin_unlock_irqrestore(&pjcpu->load_lock, flags);
 640                 }
 641
 642                 up_read(&pcpu->enable_sem);
 643         }
 644         return 0;
 645 }
 646
 647 static struct notifier_block cpufreq_notifier_block = {
 648         .notifier_call = cpufreq_interactive_notifier,
 649 };
 650
 651 static unsigned int *get_tokenized_data(const char *buf, int *num_tokens)
 652 {
 653         const char *cp;
 654         int i;
 655         int ntokens = 1;
 656         unsigned int *tokenized_data;
 657         int err = -EINVAL;
 658
 659         cp = buf;
 660         while ((cp = strpbrk(cp + 1, " :")))
 661                 ntokens++;
 662
 663         if (!(ntokens & 0x1))
 664                 goto err;
 665
 666         tokenized_data = kmalloc(ntokens * sizeof(unsigned int), GFP_KERNEL);
 667         if (!tokenized_data) {
 668                 err = -ENOMEM;
 669                 goto err;
 670         }
 671
 672         cp = buf;
 673         i = 0;
 674         while (i < ntokens) {
 675                 if (sscanf(cp, "%u", &tokenized_data[i++]) != 1)
 676                         goto err_kfree;
 677
 678                 cp = strpbrk(cp, " :");
 679                 if (!cp)
 680                         break;
 681                 cp++;
 682         }
 683
 684         if (i != ntokens)
 685                 goto err_kfree;
 686
 687         *num_tokens = ntokens;
 688         return tokenized_data;
 689
 690 err_kfree:
 691         kfree(tokenized_data);
 692 err:
 693         return ERR_PTR(err);
 694 }
 695
 696 static ssize_t show_target_loads(
 697         struct kobject *kobj, struct attribute *attr, char *buf)
 698 {
 699         int i;
 700         ssize_t ret = 0;
 701         unsigned long flags;
 702
 703         spin_lock_irqsave(&target_loads_lock, flags);
 704
 705         for (i = 0; i < ntarget_loads; i++)
 706                 ret += sprintf(buf + ret, "%u%s", target_loads[i],
 707                                i & 0x1 ? ":" : " ");
 708
 709         ret += sprintf(buf + ret, "\n");
 710         spin_unlock_irqrestore(&target_loads_lock, flags);
 711         return ret;
 712 }
 713
 714 static ssize_t store_target_loads(
 715         struct kobject *kobj, struct attribute *attr, const char *buf,
 716         size_t count)
 717 {
 718         int ntokens;
 719         unsigned int *new_target_loads = NULL;
 720         unsigned long flags;
 721
 722         new_target_loads = get_tokenized_data(buf, &ntokens);
 723         if (IS_ERR(new_target_loads))
 724                 return PTR_RET(new_target_loads);
 725
 726         spin_lock_irqsave(&target_loads_lock, flags);
 727         if (target_loads != default_target_loads)
 728                 kfree(target_loads);
 729         target_loads = new_target_loads;
 730         ntarget_loads = ntokens;
 731         spin_unlock_irqrestore(&target_loads_lock, flags);
 732         return count;
 733 }
 734
 735 static struct global_attr target_loads_attr =
 736         __ATTR(target_loads, S_IRUGO | S_IWUSR,
 737                 show_target_loads, store_target_loads);
 738
 739 static ssize_t show_above_hispeed_delay(
 740         struct kobject *kobj, struct attribute *attr, char *buf)
 741 {
 742         int i;
 743         ssize_t ret = 0;
 744         unsigned long flags;
 745
 746         spin_lock_irqsave(&above_hispeed_delay_lock, flags);
 747
 748         for (i = 0; i < nabove_hispeed_delay; i++)
 749                 ret += sprintf(buf + ret, "%u%s", above_hispeed_delay[i],
 750                                i & 0x1 ? ":" : " ");
 751
 752         ret += sprintf(buf + ret, "\n");
 753         spin_unlock_irqrestore(&above_hispeed_delay_lock, flags);
 754         return ret;
 755 }
 756
 757 static ssize_t store_above_hispeed_delay(
 758         struct kobject *kobj, struct attribute *attr, const char *buf,
 759         size_t count)
 760 {
 761         int ntokens;
 762         unsigned int *new_above_hispeed_delay = NULL;
 763         unsigned long flags;
 764
 765         new_above_hispeed_delay = get_tokenized_data(buf, &ntokens);
 766         if (IS_ERR(new_above_hispeed_delay))
 767                 return PTR_RET(new_above_hispeed_delay);
 768
 769         spin_lock_irqsave(&above_hispeed_delay_lock, flags);
 770         if (above_hispeed_delay != default_above_hispeed_delay)
 771                 kfree(above_hispeed_delay);
 772         above_hispeed_delay = new_above_hispeed_delay;
 773         nabove_hispeed_delay = ntokens;
 774         spin_unlock_irqrestore(&above_hispeed_delay_lock, flags);
 775         return count;
 776
 777 }
 778
 779 static struct global_attr above_hispeed_delay_attr =
 780         __ATTR(above_hispeed_delay, S_IRUGO | S_IWUSR,
 781                 show_above_hispeed_delay, store_above_hispeed_delay);
 782
 783 static ssize_t show_hispeed_freq(struct kobject *kobj,
 784                                  struct attribute *attr, char *buf)
 785 {
 786         return sprintf(buf, "%u\n", hispeed_freq);
 787 }
 788
 789 static ssize_t store_hispeed_freq(struct kobject *kobj,
 790                                   struct attribute *attr, const char *buf,
 791                                   size_t count)
 792 {
 793         int ret;
 794         long unsigned int val;
 795
 796         ret = strict_strtoul(buf, 0, &val);
 797         if (ret < 0)
 798                 return ret;
 799         hispeed_freq = val;
 800         return count;
 801 }
 802
 803 static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644,
 804                 show_hispeed_freq, store_hispeed_freq);
 805
 806
 807 static ssize_t show_go_hispeed_load(struct kobject *kobj,
 808                                      struct attribute *attr, char *buf)
 809 {
 810         return sprintf(buf, "%lu\n", go_hispeed_load);
 811 }
 812
 813 static ssize_t store_go_hispeed_load(struct kobject *kobj,
 814                         struct attribute *attr, const char *buf, size_t count)
 815 {
 816         int ret;
 817         unsigned long val;
 818
 819         ret = strict_strtoul(buf, 0, &val);
 820         if (ret < 0)
 821                 return ret;
 822         go_hispeed_load = val;
 823         return count;
 824 }
 825
 826 static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644,
 827                 show_go_hispeed_load, store_go_hispeed_load);
 828
 829 static ssize_t show_min_sample_time(struct kobject *kobj,
 830                                 struct attribute *attr, char *buf)
 831 {
 832         return sprintf(buf, "%lu\n", min_sample_time);
 833 }
 834
 835 static ssize_t store_min_sample_time(struct kobject *kobj,
 836                         struct attribute *attr, const char *buf, size_t count)
 837 {
 838         int ret;
 839         unsigned long val;
 840
 841         ret = strict_strtoul(buf, 0, &val);
 842         if (ret < 0)
 843                 return ret;
 844         min_sample_time = val;
 845         return count;
 846 }
 847
 848 static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
 849                 show_min_sample_time, store_min_sample_time);
 850
 851 static ssize_t show_timer_rate(struct kobject *kobj,
 852                         struct attribute *attr, char *buf)
 853 {
 854         return sprintf(buf, "%lu\n", timer_rate);
 855 }
 856
 857 static ssize_t store_timer_rate(struct kobject *kobj,
 858                         struct attribute *attr, const char *buf, size_t count)
 859 {
 860         int ret;
 861         unsigned long val;
 862
 863         ret = strict_strtoul(buf, 0, &val);
 864         if (ret < 0)
 865                 return ret;
 866         timer_rate = val;
 867         return count;
 868 }
 869
 870 static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
 871                 show_timer_rate, store_timer_rate);
 872
 873 static ssize_t show_timer_slack(
 874         struct kobject *kobj, struct attribute *attr, char *buf)
 875 {
 876         return sprintf(buf, "%d\n", timer_slack_val);
 877 }
 878
 879 static ssize_t store_timer_slack(
 880         struct kobject *kobj, struct attribute *attr, const char *buf,
 881         size_t count)
 882 {
 883         int ret;
 884         unsigned long val;
 885
 886         ret = kstrtol(buf, 10, &val);
 887         if (ret < 0)
 888                 return ret;
 889
 890         timer_slack_val = val;
 891         return count;
 892 }
 893
 894 define_one_global_rw(timer_slack);
 895
 896 static ssize_t show_boost(struct kobject *kobj, struct attribute *attr,
 897                           char *buf)
 898 {
 899         return sprintf(buf, "%d\n", boost_val);
 900 }
 901
 902 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 903                            const char *buf, size_t count)
 904 {
 905         int ret;
 906         unsigned long val;
 907
 908         ret = kstrtoul(buf, 0, &val);
 909         if (ret < 0)
 910                 return ret;
 911
 912         boost_val = val;
 913
 914         if (boost_val) {
 915                 trace_cpufreq_interactive_boost("on");
 916                 cpufreq_interactive_boost();
 917         } else {
 918                 trace_cpufreq_interactive_unboost("off");
 919         }
 920
 921         return count;
 922 }
 923
 924 define_one_global_rw(boost);
 925
 926 static ssize_t store_boostpulse(struct kobject *kobj, struct attribute *attr,
 927                                 const char *buf, size_t count)
 928 {
 929         int ret;
 930         unsigned long val;
 931
 932         ret = kstrtoul(buf, 0, &val);
 933         if (ret < 0)
 934                 return ret;
 935
 936         boostpulse_endtime = ktime_to_us(ktime_get()) + boostpulse_duration_val;
 937         trace_cpufreq_interactive_boost("pulse");
 938         cpufreq_interactive_boost();
 939         return count;
 940 }
 941
 942 static struct global_attr boostpulse =
 943         __ATTR(boostpulse, 0200, NULL, store_boostpulse);
 944
 945 static ssize_t show_boostpulse_duration(
 946         struct kobject *kobj, struct attribute *attr, char *buf)
 947 {
 948         return sprintf(buf, "%d\n", boostpulse_duration_val);
 949 }
 950
 951 static ssize_t store_boostpulse_duration(
 952         struct kobject *kobj, struct attribute *attr, const char *buf,
 953         size_t count)
 954 {
 955         int ret;
 956         unsigned long val;
 957
 958         ret = kstrtoul(buf, 0, &val);
 959         if (ret < 0)
 960                 return ret;
 961
 962         boostpulse_duration_val = val;
 963         return count;
 964 }
 965
 966 define_one_global_rw(boostpulse_duration);
 967
 968 static ssize_t show_io_is_busy(struct kobject *kobj,
 969                         struct attribute *attr, char *buf)
 970 {
 971         return sprintf(buf, "%u\n", io_is_busy);
 972 }
 973
 974 static ssize_t store_io_is_busy(struct kobject *kobj,
 975                         struct attribute *attr, const char *buf, size_t count)
 976 {
 977         int ret;
 978         unsigned long val;
 979
 980         ret = kstrtoul(buf, 0, &val);
 981         if (ret < 0)
 982                 return ret;
 983         io_is_busy = val;
 984         return count;
 985 }
 986
 987 static struct global_attr io_is_busy_attr = __ATTR(io_is_busy, 0644,
 988                 show_io_is_busy, store_io_is_busy);
 989
 990 static struct attribute *interactive_attributes[] = {
 991         &target_loads_attr.attr,
 992         &above_hispeed_delay_attr.attr,
 993         &hispeed_freq_attr.attr,
 994         &go_hispeed_load_attr.attr,
 995         &min_sample_time_attr.attr,
 996         &timer_rate_attr.attr,
 997         &timer_slack.attr,
 998         &boost.attr,
 999         &boostpulse.attr,
1000         &boostpulse_duration.attr,
1001         &io_is_busy_attr.attr,
1002         NULL,
1003 };
1004
1005 static struct attribute_group interactive_attr_group = {
1006         .attrs = interactive_attributes,
1007         .name = "interactive",
1008 };
1009
1010 static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
1011                                              unsigned long val,
1012                                              void *data)
1013 {
1014         switch (val) {
1015         case IDLE_START:
1016                 cpufreq_interactive_idle_start();
1017                 break;
1018         case IDLE_END:
1019                 cpufreq_interactive_idle_end();
1020                 break;
1021         }
1022
1023         return 0;
1024 }
1025
1026 static struct notifier_block cpufreq_interactive_idle_nb = {
1027         .notifier_call = cpufreq_interactive_idle_notifier,
1028 };
1029
1030 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
1031                 unsigned int event)
1032 {
1033         int rc;
1034         unsigned int j;
1035         struct cpufreq_interactive_cpuinfo *pcpu;
1036         struct cpufreq_frequency_table *freq_table;
1037
1038         switch (event) {
1039         case CPUFREQ_GOV_START:
1040                 if (!cpu_online(policy->cpu))
1041                         return -EINVAL;
1042
1043                 mutex_lock(&gov_lock);
1044
1045                 freq_table =
1046                         cpufreq_frequency_get_table(policy->cpu);
1047                 if (!hispeed_freq)
1048                         hispeed_freq = policy->max;
1049
1050                 for_each_cpu(j, policy->cpus) {
1051                         unsigned long expires;
1052
1053                         pcpu = &per_cpu(cpuinfo, j);
1054                         pcpu->policy = policy;
1055                         pcpu->target_freq = policy->cur;
1056                         pcpu->freq_table = freq_table;
1057                         pcpu->floor_freq = pcpu->target_freq;
1058                         pcpu->floor_validate_time =
1059                                 ktime_to_us(ktime_get());
1060                         pcpu->hispeed_validate_time =
1061                                 pcpu->floor_validate_time;
1062                         down_write(&pcpu->enable_sem);
1063                         expires = jiffies + usecs_to_jiffies(timer_rate);
1064                         pcpu->cpu_timer.expires = expires;
1065                         add_timer_on(&pcpu->cpu_timer, j);
1066                         if (timer_slack_val >= 0) {
1067                                 expires += usecs_to_jiffies(timer_slack_val);
1068                                 pcpu->cpu_slack_timer.expires = expires;
1069                                 add_timer_on(&pcpu->cpu_slack_timer, j);
1070                         }
1071                         pcpu->governor_enabled = 1;
1072                         up_write(&pcpu->enable_sem);
1073                 }
1074
1075                 /*
1076                  * Do not register the idle hook and create sysfs
1077                  * entries if we have already done so.
1078                  */
1079                 if (++active_count > 1) {
1080                         mutex_unlock(&gov_lock);
1081                         return 0;
1082                 }
1083
1084                 rc = sysfs_create_group(cpufreq_global_kobject,
1085                                 &interactive_attr_group);
1086                 if (rc) {
1087                         mutex_unlock(&gov_lock);
1088                         return rc;
1089                 }
1090
1091                 idle_notifier_register(&cpufreq_interactive_idle_nb);
1092                 cpufreq_register_notifier(
1093                         &cpufreq_notifier_block, CPUFREQ_TRANSITION_NOTIFIER);
1094                 mutex_unlock(&gov_lock);
1095                 break;
1096
1097         case CPUFREQ_GOV_STOP:
1098                 mutex_lock(&gov_lock);
1099                 for_each_cpu(j, policy->cpus) {
1100                         pcpu = &per_cpu(cpuinfo, j);
1101                         down_write(&pcpu->enable_sem);
1102                         pcpu->governor_enabled = 0;
1103                         del_timer_sync(&pcpu->cpu_timer);
1104                         del_timer_sync(&pcpu->cpu_slack_timer);
1105                         up_write(&pcpu->enable_sem);
1106                 }
1107
1108                 if (--active_count > 0) {
1109                         mutex_unlock(&gov_lock);
1110                         return 0;
1111                 }
1112
1113                 cpufreq_unregister_notifier(
1114                         &cpufreq_notifier_block, CPUFREQ_TRANSITION_NOTIFIER);
1115                 idle_notifier_unregister(&cpufreq_interactive_idle_nb);
1116                 sysfs_remove_group(cpufreq_global_kobject,
1117                                 &interactive_attr_group);
1118                 mutex_unlock(&gov_lock);
1119
1120                 break;
1121
1122         case CPUFREQ_GOV_LIMITS:
1123                 if (policy->max < policy->cur)
1124                         __cpufreq_driver_target(policy,
1125                                         policy->max, CPUFREQ_RELATION_H);
1126                 else if (policy->min > policy->cur)
1127                         __cpufreq_driver_target(policy,
1128                                         policy->min, CPUFREQ_RELATION_L);
1129                 break;
1130         }
1131         return 0;
1132 }
1133
1134 static void cpufreq_interactive_nop_timer(unsigned long data)
1135 {
1136 }
1137
1138 static int __init cpufreq_interactive_init(void)
1139 {
1140         unsigned int i;
1141         struct cpufreq_interactive_cpuinfo *pcpu;
1142         struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
1143
1144         /* Initalize per-cpu timers */
1145         for_each_possible_cpu(i) {
1146                 pcpu = &per_cpu(cpuinfo, i);
1147                 init_timer_deferrable(&pcpu->cpu_timer);
1148                 pcpu->cpu_timer.function = cpufreq_interactive_timer;
1149                 pcpu->cpu_timer.data = i;
1150                 init_timer(&pcpu->cpu_slack_timer);
1151                 pcpu->cpu_slack_timer.function = cpufreq_interactive_nop_timer;
1152                 spin_lock_init(&pcpu->load_lock);
1153                 init_rwsem(&pcpu->enable_sem);
1154         }
1155
1156         spin_lock_init(&target_loads_lock);
1157         spin_lock_init(&speedchange_cpumask_lock);
1158         spin_lock_init(&above_hispeed_delay_lock);
1159         mutex_init(&gov_lock);
1160         speedchange_task =
1161                 kthread_create(cpufreq_interactive_speedchange_task, NULL,
1162                                "cfinteractive");
1163         if (IS_ERR(speedchange_task))
1164                 return PTR_ERR(speedchange_task);
1165
1166         sched_setscheduler_nocheck(speedchange_task, SCHED_FIFO, &param);
1167         get_task_struct(speedchange_task);
1168
1169         /* NB: wake up so the thread does not look hung to the freezer */
1170         wake_up_process(speedchange_task);
1171
1172         return cpufreq_register_governor(&cpufreq_gov_interactive);
1173 }
1174
1175 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
1176 fs_initcall(cpufreq_interactive_init);
1177 #else
1178 module_init(cpufreq_interactive_init);
1179 #endif
1180
1181 static void __exit cpufreq_interactive_exit(void)
1182 {
1183         cpufreq_unregister_governor(&cpufreq_gov_interactive);
1184         kthread_stop(speedchange_task);
1185         put_task_struct(speedchange_task);
1186 }
1187
1188 module_exit(cpufreq_interactive_exit);
1189
1190 MODULE_AUTHOR("Mike Chan <mike@android.com>");
1191 MODULE_DESCRIPTION("'cpufreq_interactive' - A cpufreq governor for "
1192         "Latency sensitive workloads");
1193 MODULE_LICENSE("GPL");