drivers/cpufreq/cpufreq_interactive.c

   1 /*
   2  * drivers/cpufreq/cpufreq_interactive.c
   3  *
   4  * Copyright (C) 2010 Google, Inc.
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * Author: Mike Chan (mike@android.com)
  16  *
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/cpumask.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/module.h>
  23 #include <linux/moduleparam.h>
  24 #include <linux/rwsem.h>
  25 #include <linux/sched.h>
  26 #include <linux/sched/rt.h>
  27 #include <linux/tick.h>
  28 #include <linux/time.h>
  29 #include <linux/timer.h>
  30 #include <linux/workqueue.h>
  31 #include <linux/kthread.h>
  32 #include <linux/slab.h>
  33 #include <linux/kernel_stat.h>
  34 #include <asm/cputime.h>
  35
  36 #define CREATE_TRACE_POINTS
  37 #include <trace/events/cpufreq_interactive.h>
  38
  39 static int active_count;
  40
  41 struct cpufreq_interactive_cpuinfo {
  42         struct timer_list cpu_timer;
  43         struct timer_list cpu_slack_timer;
  44         spinlock_t load_lock; /* protects the next 4 fields */
  45         u64 time_in_idle;
  46         u64 time_in_idle_timestamp;
  47         u64 cputime_speedadj;
  48         u64 cputime_speedadj_timestamp;
  49         struct cpufreq_policy *policy;
  50         struct cpufreq_frequency_table *freq_table;
  51         unsigned int target_freq;
  52         unsigned int floor_freq;
  53         u64 floor_validate_time;
  54         u64 hispeed_validate_time;
  55         struct rw_semaphore enable_sem;
  56         int governor_enabled;
  57 };
  58
  59 static DEFINE_PER_CPU(struct cpufreq_interactive_cpuinfo, cpuinfo);
  60
  61 /* realtime thread handles frequency scaling */
  62 static struct task_struct *speedchange_task;
  63 static cpumask_t speedchange_cpumask;
  64 static spinlock_t speedchange_cpumask_lock;
  65 static struct mutex gov_lock;
  66
  67 /* Hi speed to bump to from lo speed when load burst (default max) */
  68 static unsigned int hispeed_freq;
  69
  70 /* Go to hi speed when CPU load at or above this value. */
  71 #define DEFAULT_GO_HISPEED_LOAD 99
  72 static unsigned long go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
  73
  74 /* Target load.  Lower values result in higher CPU speeds. */
  75 #define DEFAULT_TARGET_LOAD 90
  76 static unsigned int default_target_loads[] = {DEFAULT_TARGET_LOAD};
  77 static spinlock_t target_loads_lock;
  78 static unsigned int *target_loads = default_target_loads;
  79 static int ntarget_loads = ARRAY_SIZE(default_target_loads);
  80
  81 /*
  82  * The minimum amount of time to spend at a frequency before we can ramp down.
  83  */
  84 #define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
  85 static unsigned long min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
  86
  87 /*
  88  * The sample rate of the timer used to increase frequency
  89  */
  90 #define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC)
  91 static unsigned long timer_rate = DEFAULT_TIMER_RATE;
  92
  93 /*
  94  * Wait this long before raising speed above hispeed, by default a single
  95  * timer interval.
  96  */
  97 #define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE
  98 static unsigned int default_above_hispeed_delay[] = {
  99         DEFAULT_ABOVE_HISPEED_DELAY };
 100 static spinlock_t above_hispeed_delay_lock;
 101 static unsigned int *above_hispeed_delay = default_above_hispeed_delay;
 102 static int nabove_hispeed_delay = ARRAY_SIZE(default_above_hispeed_delay);
 103
 104 /* Non-zero means indefinite speed boost active */
 105 static int boost_val;
 106 /* Duration of a boot pulse in usecs */
 107 static int boostpulse_duration_val = DEFAULT_MIN_SAMPLE_TIME;
 108 /* End time of boost pulse in ktime converted to usecs */
 109 static u64 boostpulse_endtime;
 110
 111 /*
 112  * Max additional time to wait in idle, beyond timer_rate, at speeds above
 113  * minimum before wakeup to reduce speed, or -1 if unnecessary.
 114  */
 115 #define DEFAULT_TIMER_SLACK (4 * DEFAULT_TIMER_RATE)
 116 static int timer_slack_val = DEFAULT_TIMER_SLACK;
 117
 118 static bool io_is_busy;
 119
 120 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 121                 unsigned int event);
 122
 123 #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 124 static
 125 #endif
 126 struct cpufreq_governor cpufreq_gov_interactive = {
 127         .name = "interactive",
 128         .governor = cpufreq_governor_interactive,
 129         .max_transition_latency = 10000000,
 130         .owner = THIS_MODULE,
 131 };
 132
 133 static inline cputime64_t get_cpu_idle_time_jiffy(unsigned int cpu,
 134                                                   cputime64_t *wall)
 135 {
 136         u64 idle_time;
 137         u64 cur_wall_time;
 138         u64 busy_time;
 139
 140         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 141
 142         busy_time  = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 143         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 144         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 145         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 146         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 147         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 148
 149         idle_time = cur_wall_time - busy_time;
 150         if (wall)
 151                 *wall = jiffies_to_usecs(cur_wall_time);
 152
 153         return jiffies_to_usecs(idle_time);
 154 }
 155
 156 static inline cputime64_t get_cpu_idle_time(unsigned int cpu,
 157                                             cputime64_t *wall)
 158 {
 159         u64 idle_time = get_cpu_idle_time_us(cpu, wall);
 160
 161         if (idle_time == -1ULL)
 162                 idle_time = get_cpu_idle_time_jiffy(cpu, wall);
 163         else if (!io_is_busy)
 164                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 165
 166         return idle_time;
 167 }
 168
 169 static void cpufreq_interactive_timer_resched(
 170         struct cpufreq_interactive_cpuinfo *pcpu)
 171 {
 172         unsigned long expires;
 173         unsigned long flags;
 174
 175         spin_lock_irqsave(&pcpu->load_lock, flags);
 176         pcpu->time_in_idle =
 177                 get_cpu_idle_time(smp_processor_id(),
 178                                      &pcpu->time_in_idle_timestamp);
 179         pcpu->cputime_speedadj = 0;
 180         pcpu->cputime_speedadj_timestamp = pcpu->time_in_idle_timestamp;
 181         expires = jiffies + usecs_to_jiffies(timer_rate);
 182         mod_timer_pinned(&pcpu->cpu_timer, expires);
 183
 184         if (timer_slack_val >= 0 && pcpu->target_freq > pcpu->policy->min) {
 185                 expires += usecs_to_jiffies(timer_slack_val);
 186                 mod_timer_pinned(&pcpu->cpu_slack_timer, expires);
 187         }
 188
 189         spin_unlock_irqrestore(&pcpu->load_lock, flags);
 190 }
 191
 192 /* The caller shall take enable_sem write semaphore to avoid any timer race.
 193  * The cpu_timer and cpu_slack_timer must be deactivated when calling this
 194  * function.
 195  */
 196 static void cpufreq_interactive_timer_start(int cpu)
 197 {
 198         struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, cpu);
 199         unsigned long expires = jiffies + usecs_to_jiffies(timer_rate);
 200         unsigned long flags;
 201
 202         pcpu->cpu_timer.expires = expires;
 203         add_timer_on(&pcpu->cpu_timer, cpu);
 204         if (timer_slack_val >= 0 && pcpu->target_freq > pcpu->policy->min) {
 205                 expires += usecs_to_jiffies(timer_slack_val);
 206                 pcpu->cpu_slack_timer.expires = expires;
 207                 add_timer_on(&pcpu->cpu_slack_timer, cpu);
 208         }
 209
 210         spin_lock_irqsave(&pcpu->load_lock, flags);
 211         pcpu->time_in_idle =
 212                 get_cpu_idle_time(cpu, &pcpu->time_in_idle_timestamp);
 213         pcpu->cputime_speedadj = 0;
 214         pcpu->cputime_speedadj_timestamp = pcpu->time_in_idle_timestamp;
 215         spin_unlock_irqrestore(&pcpu->load_lock, flags);
 216 }
 217
 218 static unsigned int freq_to_above_hispeed_delay(unsigned int freq)
 219 {
 220         int i;
 221         unsigned int ret;
 222         unsigned long flags;
 223
 224         spin_lock_irqsave(&above_hispeed_delay_lock, flags);
 225
 226         for (i = 0; i < nabove_hispeed_delay - 1 &&
 227                         freq >= above_hispeed_delay[i+1]; i += 2)
 228                 ;
 229
 230         ret = above_hispeed_delay[i];
 231         spin_unlock_irqrestore(&above_hispeed_delay_lock, flags);
 232         return ret;
 233 }
 234
 235 static unsigned int freq_to_targetload(unsigned int freq)
 236 {
 237         int i;
 238         unsigned int ret;
 239         unsigned long flags;
 240
 241         spin_lock_irqsave(&target_loads_lock, flags);
 242
 243         for (i = 0; i < ntarget_loads - 1 && freq >= target_loads[i+1]; i += 2)
 244                 ;
 245
 246         ret = target_loads[i];
 247         spin_unlock_irqrestore(&target_loads_lock, flags);
 248         return ret;
 249 }
 250
 251 /*
 252  * If increasing frequencies never map to a lower target load then
 253  * choose_freq() will find the minimum frequency that does not exceed its
 254  * target load given the current load.
 255  */
 256
 257 static unsigned int choose_freq(
 258         struct cpufreq_interactive_cpuinfo *pcpu, unsigned int loadadjfreq)
 259 {
 260         unsigned int freq = pcpu->policy->cur;
 261         unsigned int prevfreq, freqmin, freqmax;
 262         unsigned int tl;
 263         int index;
 264
 265         freqmin = 0;
 266         freqmax = UINT_MAX;
 267
 268         do {
 269                 prevfreq = freq;
 270                 tl = freq_to_targetload(freq);
 271
 272                 /*
 273                  * Find the lowest frequency where the computed load is less
 274                  * than or equal to the target load.
 275                  */
 276
 277                 if (cpufreq_frequency_table_target(
 278                             pcpu->policy, pcpu->freq_table, loadadjfreq / tl,
 279                             CPUFREQ_RELATION_L, &index))
 280                         break;
 281                 freq = pcpu->freq_table[index].frequency;
 282
 283                 if (freq > prevfreq) {
 284                         /* The previous frequency is too low. */
 285                         freqmin = prevfreq;
 286
 287                         if (freq >= freqmax) {
 288                                 /*
 289                                  * Find the highest frequency that is less
 290                                  * than freqmax.
 291                                  */
 292                                 if (cpufreq_frequency_table_target(
 293                                             pcpu->policy, pcpu->freq_table,
 294                                             freqmax - 1, CPUFREQ_RELATION_H,
 295                                             &index))
 296                                         break;
 297                                 freq = pcpu->freq_table[index].frequency;
 298
 299                                 if (freq == freqmin) {
 300                                         /*
 301                                          * The first frequency below freqmax
 302                                          * has already been found to be too
 303                                          * low.  freqmax is the lowest speed
 304                                          * we found that is fast enough.
 305                                          */
 306                                         freq = freqmax;
 307                                         break;
 308                                 }
 309                         }
 310                 } else if (freq < prevfreq) {
 311                         /* The previous frequency is high enough. */
 312                         freqmax = prevfreq;
 313
 314                         if (freq <= freqmin) {
 315                                 /*
 316                                  * Find the lowest frequency that is higher
 317                                  * than freqmin.
 318                                  */
 319                                 if (cpufreq_frequency_table_target(
 320                                             pcpu->policy, pcpu->freq_table,
 321                                             freqmin + 1, CPUFREQ_RELATION_L,
 322                                             &index))
 323                                         break;
 324                                 freq = pcpu->freq_table[index].frequency;
 325
 326                                 /*
 327                                  * If freqmax is the first frequency above
 328                                  * freqmin then we have already found that
 329                                  * this speed is fast enough.
 330                                  */
 331                                 if (freq == freqmax)
 332                                         break;
 333                         }
 334                 }
 335
 336                 /* If same frequency chosen as previous then done. */
 337         } while (freq != prevfreq);
 338
 339         return freq;
 340 }
 341
 342 static u64 update_load(int cpu)
 343 {
 344         struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, cpu);
 345         u64 now;
 346         u64 now_idle;
 347         unsigned int delta_idle;
 348         unsigned int delta_time;
 349         u64 active_time;
 350
 351         now_idle = get_cpu_idle_time(cpu, &now);
 352         delta_idle = (unsigned int)(now_idle - pcpu->time_in_idle);
 353         delta_time = (unsigned int)(now - pcpu->time_in_idle_timestamp);
 354
 355         if (delta_time <= delta_idle)
 356                 active_time = 0;
 357         else
 358                 active_time = delta_time - delta_idle;
 359
 360         pcpu->cputime_speedadj += active_time * pcpu->policy->cur;
 361
 362         pcpu->time_in_idle = now_idle;
 363         pcpu->time_in_idle_timestamp = now;
 364         return now;
 365 }
 366
 367 static void cpufreq_interactive_timer(unsigned long data)
 368 {
 369         u64 now;
 370         unsigned int delta_time;
 371         u64 cputime_speedadj;
 372         int cpu_load;
 373         struct cpufreq_interactive_cpuinfo *pcpu =
 374                 &per_cpu(cpuinfo, data);
 375         unsigned int new_freq;
 376         unsigned int loadadjfreq;
 377         unsigned int index;
 378         unsigned long flags;
 379         bool boosted;
 380
 381         if (!down_read_trylock(&pcpu->enable_sem))
 382                 return;
 383         if (!pcpu->governor_enabled)
 384                 goto exit;
 385
 386         spin_lock_irqsave(&pcpu->load_lock, flags);
 387         now = update_load(data);
 388         delta_time = (unsigned int)(now - pcpu->cputime_speedadj_timestamp);
 389         cputime_speedadj = pcpu->cputime_speedadj;
 390         spin_unlock_irqrestore(&pcpu->load_lock, flags);
 391
 392         if (WARN_ON_ONCE(!delta_time))
 393                 goto rearm;
 394
 395         do_div(cputime_speedadj, delta_time);
 396         loadadjfreq = (unsigned int)cputime_speedadj * 100;
 397         cpu_load = loadadjfreq / pcpu->target_freq;
 398         boosted = boost_val || now < boostpulse_endtime;
 399
 400         if (cpu_load >= go_hispeed_load || boosted) {
 401                 if (pcpu->target_freq < hispeed_freq) {
 402                         new_freq = hispeed_freq;
 403                 } else {
 404                         new_freq = choose_freq(pcpu, loadadjfreq);
 405
 406                         if (new_freq < hispeed_freq)
 407                                 new_freq = hispeed_freq;
 408                 }
 409         } else {
 410                 new_freq = choose_freq(pcpu, loadadjfreq);
 411         }
 412
 413         if (pcpu->target_freq >= hispeed_freq &&
 414             new_freq > pcpu->target_freq &&
 415             now - pcpu->hispeed_validate_time <
 416             freq_to_above_hispeed_delay(pcpu->target_freq)) {
 417                 trace_cpufreq_interactive_notyet(
 418                         data, cpu_load, pcpu->target_freq,
 419                         pcpu->policy->cur, new_freq);
 420                 goto rearm;
 421         }
 422
 423         pcpu->hispeed_validate_time = now;
 424
 425         if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
 426                                            new_freq, CPUFREQ_RELATION_L,
 427                                            &index))
 428                 goto rearm;
 429
 430         new_freq = pcpu->freq_table[index].frequency;
 431
 432         /*
 433          * Do not scale below floor_freq unless we have been at or above the
 434          * floor frequency for the minimum sample time since last validated.
 435          */
 436         if (new_freq < pcpu->floor_freq) {
 437                 if (now - pcpu->floor_validate_time < min_sample_time) {
 438                         trace_cpufreq_interactive_notyet(
 439                                 data, cpu_load, pcpu->target_freq,
 440                                 pcpu->policy->cur, new_freq);
 441                         goto rearm;
 442                 }
 443         }
 444
 445         /*
 446          * Update the timestamp for checking whether speed has been held at
 447          * or above the selected frequency for a minimum of min_sample_time,
 448          * if not boosted to hispeed_freq.  If boosted to hispeed_freq then we
 449          * allow the speed to drop as soon as the boostpulse duration expires
 450          * (or the indefinite boost is turned off).
 451          */
 452
 453         if (!boosted || new_freq > hispeed_freq) {
 454                 pcpu->floor_freq = new_freq;
 455                 pcpu->floor_validate_time = now;
 456         }
 457
 458         if (pcpu->target_freq == new_freq) {
 459                 trace_cpufreq_interactive_already(
 460                         data, cpu_load, pcpu->target_freq,
 461                         pcpu->policy->cur, new_freq);
 462                 goto rearm_if_notmax;
 463         }
 464
 465         trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
 466                                          pcpu->policy->cur, new_freq);
 467
 468         pcpu->target_freq = new_freq;
 469         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 470         cpumask_set_cpu(data, &speedchange_cpumask);
 471         spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 472         wake_up_process(speedchange_task);
 473
 474 rearm_if_notmax:
 475         /*
 476          * Already set max speed and don't see a need to change that,
 477          * wait until next idle to re-evaluate, don't need timer.
 478          */
 479         if (pcpu->target_freq == pcpu->policy->max)
 480                 goto exit;
 481
 482 rearm:
 483         if (!timer_pending(&pcpu->cpu_timer))
 484                 cpufreq_interactive_timer_resched(pcpu);
 485
 486 exit:
 487         up_read(&pcpu->enable_sem);
 488         return;
 489 }
 490
 491 static void cpufreq_interactive_idle_start(void)
 492 {
 493         struct cpufreq_interactive_cpuinfo *pcpu =
 494                 &per_cpu(cpuinfo, smp_processor_id());
 495         int pending;
 496
 497         if (!down_read_trylock(&pcpu->enable_sem))
 498                 return;
 499         if (!pcpu->governor_enabled) {
 500                 up_read(&pcpu->enable_sem);
 501                 return;
 502         }
 503
 504         pending = timer_pending(&pcpu->cpu_timer);
 505
 506         if (pcpu->target_freq != pcpu->policy->min) {
 507                 /*
 508                  * Entering idle while not at lowest speed.  On some
 509                  * platforms this can hold the other CPU(s) at that speed
 510                  * even though the CPU is idle. Set a timer to re-evaluate
 511                  * speed so this idle CPU doesn't hold the other CPUs above
 512                  * min indefinitely.  This should probably be a quirk of
 513                  * the CPUFreq driver.
 514                  */
 515                 if (!pending)
 516                         cpufreq_interactive_timer_resched(pcpu);
 517         }
 518
 519         up_read(&pcpu->enable_sem);
 520 }
 521
 522 static void cpufreq_interactive_idle_end(void)
 523 {
 524         struct cpufreq_interactive_cpuinfo *pcpu =
 525                 &per_cpu(cpuinfo, smp_processor_id());
 526
 527         if (!down_read_trylock(&pcpu->enable_sem))
 528                 return;
 529         if (!pcpu->governor_enabled) {
 530                 up_read(&pcpu->enable_sem);
 531                 return;
 532         }
 533
 534         /* Arm the timer for 1-2 ticks later if not already. */
 535         if (!timer_pending(&pcpu->cpu_timer)) {
 536                 cpufreq_interactive_timer_resched(pcpu);
 537         } else if (time_after_eq(jiffies, pcpu->cpu_timer.expires)) {
 538                 del_timer(&pcpu->cpu_timer);
 539                 del_timer(&pcpu->cpu_slack_timer);
 540                 cpufreq_interactive_timer(smp_processor_id());
 541         }
 542
 543         up_read(&pcpu->enable_sem);
 544 }
 545
 546 static int cpufreq_interactive_speedchange_task(void *data)
 547 {
 548         unsigned int cpu;
 549         cpumask_t tmp_mask;
 550         unsigned long flags;
 551         struct cpufreq_interactive_cpuinfo *pcpu;
 552
 553         while (1) {
 554                 set_current_state(TASK_INTERRUPTIBLE);
 555                 spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 556
 557                 if (cpumask_empty(&speedchange_cpumask)) {
 558                         spin_unlock_irqrestore(&speedchange_cpumask_lock,
 559                                                flags);
 560                         schedule();
 561
 562                         if (kthread_should_stop())
 563                                 break;
 564
 565                         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 566                 }
 567
 568                 set_current_state(TASK_RUNNING);
 569                 tmp_mask = speedchange_cpumask;
 570                 cpumask_clear(&speedchange_cpumask);
 571                 spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 572
 573                 for_each_cpu(cpu, &tmp_mask) {
 574                         unsigned int j;
 575                         unsigned int max_freq = 0;
 576
 577                         pcpu = &per_cpu(cpuinfo, cpu);
 578                         if (!down_read_trylock(&pcpu->enable_sem))
 579                                 continue;
 580                         if (!pcpu->governor_enabled) {
 581                                 up_read(&pcpu->enable_sem);
 582                                 continue;
 583                         }
 584
 585                         for_each_cpu(j, pcpu->policy->cpus) {
 586                                 struct cpufreq_interactive_cpuinfo *pjcpu =
 587                                         &per_cpu(cpuinfo, j);
 588
 589                                 if (pjcpu->target_freq > max_freq)
 590                                         max_freq = pjcpu->target_freq;
 591                         }
 592
 593                         if (max_freq != pcpu->policy->cur)
 594                                 __cpufreq_driver_target(pcpu->policy,
 595                                                         max_freq,
 596                                                         CPUFREQ_RELATION_H);
 597                         trace_cpufreq_interactive_setspeed(cpu,
 598                                                      pcpu->target_freq,
 599                                                      pcpu->policy->cur);
 600
 601                         up_read(&pcpu->enable_sem);
 602                 }
 603         }
 604
 605         return 0;
 606 }
 607
 608 static void cpufreq_interactive_boost(void)
 609 {
 610         int i;
 611         int anyboost = 0;
 612         unsigned long flags;
 613         struct cpufreq_interactive_cpuinfo *pcpu;
 614
 615         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 616
 617         for_each_online_cpu(i) {
 618                 pcpu = &per_cpu(cpuinfo, i);
 619
 620                 if (pcpu->target_freq < hispeed_freq) {
 621                         pcpu->target_freq = hispeed_freq;
 622                         cpumask_set_cpu(i, &speedchange_cpumask);
 623                         pcpu->hispeed_validate_time =
 624                                 ktime_to_us(ktime_get());
 625                         anyboost = 1;
 626                 }
 627
 628                 /*
 629                  * Set floor freq and (re)start timer for when last
 630                  * validated.
 631                  */
 632
 633                 pcpu->floor_freq = hispeed_freq;
 634                 pcpu->floor_validate_time = ktime_to_us(ktime_get());
 635         }
 636
 637         spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 638
 639         if (anyboost)
 640                 wake_up_process(speedchange_task);
 641 }
 642
 643 static int cpufreq_interactive_notifier(
 644         struct notifier_block *nb, unsigned long val, void *data)
 645 {
 646         struct cpufreq_freqs *freq = data;
 647         struct cpufreq_interactive_cpuinfo *pcpu;
 648         int cpu;
 649         unsigned long flags;
 650
 651         if (val == CPUFREQ_POSTCHANGE) {
 652                 pcpu = &per_cpu(cpuinfo, freq->cpu);
 653                 if (!down_read_trylock(&pcpu->enable_sem))
 654                         return 0;
 655                 if (!pcpu->governor_enabled) {
 656                         up_read(&pcpu->enable_sem);
 657                         return 0;
 658                 }
 659
 660                 for_each_cpu(cpu, pcpu->policy->cpus) {
 661                         struct cpufreq_interactive_cpuinfo *pjcpu =
 662                                 &per_cpu(cpuinfo, cpu);
 663                         if (cpu != freq->cpu) {
 664                                 if (!down_read_trylock(&pjcpu->enable_sem))
 665                                         continue;
 666                                 if (!pjcpu->governor_enabled) {
 667                                         up_read(&pjcpu->enable_sem);
 668                                         continue;
 669                                 }
 670                         }
 671                         spin_lock_irqsave(&pjcpu->load_lock, flags);
 672                         update_load(cpu);
 673                         spin_unlock_irqrestore(&pjcpu->load_lock, flags);
 674                         if (cpu != freq->cpu)
 675                                 up_read(&pjcpu->enable_sem);
 676                 }
 677
 678                 up_read(&pcpu->enable_sem);
 679         }
 680         return 0;
 681 }
 682
 683 static struct notifier_block cpufreq_notifier_block = {
 684         .notifier_call = cpufreq_interactive_notifier,
 685 };
 686
 687 static unsigned int *get_tokenized_data(const char *buf, int *num_tokens)
 688 {
 689         const char *cp;
 690         int i;
 691         int ntokens = 1;
 692         unsigned int *tokenized_data;
 693         int err = -EINVAL;
 694
 695         cp = buf;
 696         while ((cp = strpbrk(cp + 1, " :")))
 697                 ntokens++;
 698
 699         if (!(ntokens & 0x1))
 700                 goto err;
 701
 702         tokenized_data = kmalloc(ntokens * sizeof(unsigned int), GFP_KERNEL);
 703         if (!tokenized_data) {
 704                 err = -ENOMEM;
 705                 goto err;
 706         }
 707
 708         cp = buf;
 709         i = 0;
 710         while (i < ntokens) {
 711                 if (sscanf(cp, "%u", &tokenized_data[i++]) != 1)
 712                         goto err_kfree;
 713
 714                 cp = strpbrk(cp, " :");
 715                 if (!cp)
 716                         break;
 717                 cp++;
 718         }
 719
 720         if (i != ntokens)
 721                 goto err_kfree;
 722
 723         *num_tokens = ntokens;
 724         return tokenized_data;
 725
 726 err_kfree:
 727         kfree(tokenized_data);
 728 err:
 729         return ERR_PTR(err);
 730 }
 731
 732 static ssize_t show_target_loads(
 733         struct kobject *kobj, struct attribute *attr, char *buf)
 734 {
 735         int i;
 736         ssize_t ret = 0;
 737         unsigned long flags;
 738
 739         spin_lock_irqsave(&target_loads_lock, flags);
 740
 741         for (i = 0; i < ntarget_loads; i++)
 742                 ret += sprintf(buf + ret, "%u%s", target_loads[i],
 743                                i & 0x1 ? ":" : " ");
 744
 745         ret += sprintf(buf + --ret, "\n");
 746         spin_unlock_irqrestore(&target_loads_lock, flags);
 747         return ret;
 748 }
 749
 750 static ssize_t store_target_loads(
 751         struct kobject *kobj, struct attribute *attr, const char *buf,
 752         size_t count)
 753 {
 754         int ntokens;
 755         unsigned int *new_target_loads = NULL;
 756         unsigned long flags;
 757
 758         new_target_loads = get_tokenized_data(buf, &ntokens);
 759         if (IS_ERR(new_target_loads))
 760                 return PTR_RET(new_target_loads);
 761
 762         spin_lock_irqsave(&target_loads_lock, flags);
 763         if (target_loads != default_target_loads)
 764                 kfree(target_loads);
 765         target_loads = new_target_loads;
 766         ntarget_loads = ntokens;
 767         spin_unlock_irqrestore(&target_loads_lock, flags);
 768         return count;
 769 }
 770
 771 static struct global_attr target_loads_attr =
 772         __ATTR(target_loads, S_IRUGO | S_IWUSR,
 773                 show_target_loads, store_target_loads);
 774
 775 static ssize_t show_above_hispeed_delay(
 776         struct kobject *kobj, struct attribute *attr, char *buf)
 777 {
 778         int i;
 779         ssize_t ret = 0;
 780         unsigned long flags;
 781
 782         spin_lock_irqsave(&above_hispeed_delay_lock, flags);
 783
 784         for (i = 0; i < nabove_hispeed_delay; i++)
 785                 ret += sprintf(buf + ret, "%u%s", above_hispeed_delay[i],
 786                                i & 0x1 ? ":" : " ");
 787
 788         ret += sprintf(buf + --ret, "\n");
 789         spin_unlock_irqrestore(&above_hispeed_delay_lock, flags);
 790         return ret;
 791 }
 792
 793 static ssize_t store_above_hispeed_delay(
 794         struct kobject *kobj, struct attribute *attr, const char *buf,
 795         size_t count)
 796 {
 797         int ntokens;
 798         unsigned int *new_above_hispeed_delay = NULL;
 799         unsigned long flags;
 800
 801         new_above_hispeed_delay = get_tokenized_data(buf, &ntokens);
 802         if (IS_ERR(new_above_hispeed_delay))
 803                 return PTR_RET(new_above_hispeed_delay);
 804
 805         spin_lock_irqsave(&above_hispeed_delay_lock, flags);
 806         if (above_hispeed_delay != default_above_hispeed_delay)
 807                 kfree(above_hispeed_delay);
 808         above_hispeed_delay = new_above_hispeed_delay;
 809         nabove_hispeed_delay = ntokens;
 810         spin_unlock_irqrestore(&above_hispeed_delay_lock, flags);
 811         return count;
 812
 813 }
 814
 815 static struct global_attr above_hispeed_delay_attr =
 816         __ATTR(above_hispeed_delay, S_IRUGO | S_IWUSR,
 817                 show_above_hispeed_delay, store_above_hispeed_delay);
 818
 819 static ssize_t show_hispeed_freq(struct kobject *kobj,
 820                                  struct attribute *attr, char *buf)
 821 {
 822         return sprintf(buf, "%u\n", hispeed_freq);
 823 }
 824
 825 static ssize_t store_hispeed_freq(struct kobject *kobj,
 826                                   struct attribute *attr, const char *buf,
 827                                   size_t count)
 828 {
 829         int ret;
 830         long unsigned int val;
 831
 832         ret = strict_strtoul(buf, 0, &val);
 833         if (ret < 0)
 834                 return ret;
 835         hispeed_freq = val;
 836         return count;
 837 }
 838
 839 static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644,
 840                 show_hispeed_freq, store_hispeed_freq);
 841
 842
 843 static ssize_t show_go_hispeed_load(struct kobject *kobj,
 844                                      struct attribute *attr, char *buf)
 845 {
 846         return sprintf(buf, "%lu\n", go_hispeed_load);
 847 }
 848
 849 static ssize_t store_go_hispeed_load(struct kobject *kobj,
 850                         struct attribute *attr, const char *buf, size_t count)
 851 {
 852         int ret;
 853         unsigned long val;
 854
 855         ret = strict_strtoul(buf, 0, &val);
 856         if (ret < 0)
 857                 return ret;
 858         go_hispeed_load = val;
 859         return count;
 860 }
 861
 862 static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644,
 863                 show_go_hispeed_load, store_go_hispeed_load);
 864
 865 static ssize_t show_min_sample_time(struct kobject *kobj,
 866                                 struct attribute *attr, char *buf)
 867 {
 868         return sprintf(buf, "%lu\n", min_sample_time);
 869 }
 870
 871 static ssize_t store_min_sample_time(struct kobject *kobj,
 872                         struct attribute *attr, const char *buf, size_t count)
 873 {
 874         int ret;
 875         unsigned long val;
 876
 877         ret = strict_strtoul(buf, 0, &val);
 878         if (ret < 0)
 879                 return ret;
 880         min_sample_time = val;
 881         return count;
 882 }
 883
 884 static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
 885                 show_min_sample_time, store_min_sample_time);
 886
 887 static ssize_t show_timer_rate(struct kobject *kobj,
 888                         struct attribute *attr, char *buf)
 889 {
 890         return sprintf(buf, "%lu\n", timer_rate);
 891 }
 892
 893 static ssize_t store_timer_rate(struct kobject *kobj,
 894                         struct attribute *attr, const char *buf, size_t count)
 895 {
 896         int ret;
 897         unsigned long val;
 898
 899         ret = strict_strtoul(buf, 0, &val);
 900         if (ret < 0)
 901                 return ret;
 902         timer_rate = val;
 903         return count;
 904 }
 905
 906 static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
 907                 show_timer_rate, store_timer_rate);
 908
 909 static ssize_t show_timer_slack(
 910         struct kobject *kobj, struct attribute *attr, char *buf)
 911 {
 912         return sprintf(buf, "%d\n", timer_slack_val);
 913 }
 914
 915 static ssize_t store_timer_slack(
 916         struct kobject *kobj, struct attribute *attr, const char *buf,
 917         size_t count)
 918 {
 919         int ret;
 920         unsigned long val;
 921
 922         ret = kstrtol(buf, 10, &val);
 923         if (ret < 0)
 924                 return ret;
 925
 926         timer_slack_val = val;
 927         return count;
 928 }
 929
 930 define_one_global_rw(timer_slack);
 931
 932 static ssize_t show_boost(struct kobject *kobj, struct attribute *attr,
 933                           char *buf)
 934 {
 935         return sprintf(buf, "%d\n", boost_val);
 936 }
 937
 938 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 939                            const char *buf, size_t count)
 940 {
 941         int ret;
 942         unsigned long val;
 943
 944         ret = kstrtoul(buf, 0, &val);
 945         if (ret < 0)
 946                 return ret;
 947
 948         boost_val = val;
 949
 950         if (boost_val) {
 951                 trace_cpufreq_interactive_boost("on");
 952                 cpufreq_interactive_boost();
 953         } else {
 954                 trace_cpufreq_interactive_unboost("off");
 955         }
 956
 957         return count;
 958 }
 959
 960 define_one_global_rw(boost);
 961
 962 static ssize_t store_boostpulse(struct kobject *kobj, struct attribute *attr,
 963                                 const char *buf, size_t count)
 964 {
 965         int ret;
 966         unsigned long val;
 967
 968         ret = kstrtoul(buf, 0, &val);
 969         if (ret < 0)
 970                 return ret;
 971
 972         boostpulse_endtime = ktime_to_us(ktime_get()) + boostpulse_duration_val;
 973         trace_cpufreq_interactive_boost("pulse");
 974         cpufreq_interactive_boost();
 975         return count;
 976 }
 977
 978 static struct global_attr boostpulse =
 979         __ATTR(boostpulse, 0200, NULL, store_boostpulse);
 980
 981 static ssize_t show_boostpulse_duration(
 982         struct kobject *kobj, struct attribute *attr, char *buf)
 983 {
 984         return sprintf(buf, "%d\n", boostpulse_duration_val);
 985 }
 986
 987 static ssize_t store_boostpulse_duration(
 988         struct kobject *kobj, struct attribute *attr, const char *buf,
 989         size_t count)
 990 {
 991         int ret;
 992         unsigned long val;
 993
 994         ret = kstrtoul(buf, 0, &val);
 995         if (ret < 0)
 996                 return ret;
 997
 998         boostpulse_duration_val = val;
 999         return count;
1000 }
1001
1002 define_one_global_rw(boostpulse_duration);
1003
1004 static ssize_t show_io_is_busy(struct kobject *kobj,
1005                         struct attribute *attr, char *buf)
1006 {
1007         return sprintf(buf, "%u\n", io_is_busy);
1008 }
1009
1010 static ssize_t store_io_is_busy(struct kobject *kobj,
1011                         struct attribute *attr, const char *buf, size_t count)
1012 {
1013         int ret;
1014         unsigned long val;
1015
1016         ret = kstrtoul(buf, 0, &val);
1017         if (ret < 0)
1018                 return ret;
1019         io_is_busy = val;
1020         return count;
1021 }
1022
1023 static struct global_attr io_is_busy_attr = __ATTR(io_is_busy, 0644,
1024                 show_io_is_busy, store_io_is_busy);
1025
1026 static struct attribute *interactive_attributes[] = {
1027         &target_loads_attr.attr,
1028         &above_hispeed_delay_attr.attr,
1029         &hispeed_freq_attr.attr,
1030         &go_hispeed_load_attr.attr,
1031         &min_sample_time_attr.attr,
1032         &timer_rate_attr.attr,
1033         &timer_slack.attr,
1034         &boost.attr,
1035         &boostpulse.attr,
1036         &boostpulse_duration.attr,
1037         &io_is_busy_attr.attr,
1038         NULL,
1039 };
1040
1041 static struct attribute_group interactive_attr_group = {
1042         .attrs = interactive_attributes,
1043         .name = "interactive",
1044 };
1045
1046 static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
1047                                              unsigned long val,
1048                                              void *data)
1049 {
1050         switch (val) {
1051         case IDLE_START:
1052                 cpufreq_interactive_idle_start();
1053                 break;
1054         case IDLE_END:
1055                 cpufreq_interactive_idle_end();
1056                 break;
1057         }
1058
1059         return 0;
1060 }
1061
1062 static struct notifier_block cpufreq_interactive_idle_nb = {
1063         .notifier_call = cpufreq_interactive_idle_notifier,
1064 };
1065
1066 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
1067                 unsigned int event)
1068 {
1069         int rc;
1070         unsigned int j;
1071         struct cpufreq_interactive_cpuinfo *pcpu;
1072         struct cpufreq_frequency_table *freq_table;
1073
1074         switch (event) {
1075         case CPUFREQ_GOV_START:
1076                 if (!cpu_online(policy->cpu))
1077                         return -EINVAL;
1078
1079                 mutex_lock(&gov_lock);
1080
1081                 freq_table =
1082                         cpufreq_frequency_get_table(policy->cpu);
1083                 if (!hispeed_freq)
1084                         hispeed_freq = policy->max;
1085
1086                 for_each_cpu(j, policy->cpus) {
1087                         pcpu = &per_cpu(cpuinfo, j);
1088                         pcpu->policy = policy;
1089                         pcpu->target_freq = policy->cur;
1090                         pcpu->freq_table = freq_table;
1091                         pcpu->floor_freq = pcpu->target_freq;
1092                         pcpu->floor_validate_time =
1093                                 ktime_to_us(ktime_get());
1094                         pcpu->hispeed_validate_time =
1095                                 pcpu->floor_validate_time;
1096                         down_write(&pcpu->enable_sem);
1097                         cpufreq_interactive_timer_start(j);
1098                         pcpu->governor_enabled = 1;
1099                         up_write(&pcpu->enable_sem);
1100                 }
1101
1102                 /*
1103                  * Do not register the idle hook and create sysfs
1104                  * entries if we have already done so.
1105                  */
1106                 if (++active_count > 1) {
1107                         mutex_unlock(&gov_lock);
1108                         return 0;
1109                 }
1110
1111                 rc = sysfs_create_group(cpufreq_global_kobject,
1112                                 &interactive_attr_group);
1113                 if (rc) {
1114                         mutex_unlock(&gov_lock);
1115                         return rc;
1116                 }
1117
1118                 idle_notifier_register(&cpufreq_interactive_idle_nb);
1119                 cpufreq_register_notifier(
1120                         &cpufreq_notifier_block, CPUFREQ_TRANSITION_NOTIFIER);
1121                 mutex_unlock(&gov_lock);
1122                 break;
1123
1124         case CPUFREQ_GOV_STOP:
1125                 mutex_lock(&gov_lock);
1126                 for_each_cpu(j, policy->cpus) {
1127                         pcpu = &per_cpu(cpuinfo, j);
1128                         down_write(&pcpu->enable_sem);
1129                         pcpu->governor_enabled = 0;
1130                         del_timer_sync(&pcpu->cpu_timer);
1131                         del_timer_sync(&pcpu->cpu_slack_timer);
1132                         up_write(&pcpu->enable_sem);
1133                 }
1134
1135                 if (--active_count > 0) {
1136                         mutex_unlock(&gov_lock);
1137                         return 0;
1138                 }
1139
1140                 cpufreq_unregister_notifier(
1141                         &cpufreq_notifier_block, CPUFREQ_TRANSITION_NOTIFIER);
1142                 idle_notifier_unregister(&cpufreq_interactive_idle_nb);
1143                 sysfs_remove_group(cpufreq_global_kobject,
1144                                 &interactive_attr_group);
1145                 mutex_unlock(&gov_lock);
1146
1147                 break;
1148
1149         case CPUFREQ_GOV_LIMITS:
1150                 if (policy->max < policy->cur)
1151                         __cpufreq_driver_target(policy,
1152                                         policy->max, CPUFREQ_RELATION_H);
1153                 else if (policy->min > policy->cur)
1154                         __cpufreq_driver_target(policy,
1155                                         policy->min, CPUFREQ_RELATION_L);
1156                 for_each_cpu(j, policy->cpus) {
1157                         pcpu = &per_cpu(cpuinfo, j);
1158
1159                         /* hold write semaphore to avoid race */
1160                         down_write(&pcpu->enable_sem);
1161                         if (pcpu->governor_enabled == 0) {
1162                                 up_write(&pcpu->enable_sem);
1163                                 continue;
1164                         }
1165
1166                         /* update target_freq firstly */
1167                         if (policy->max < pcpu->target_freq)
1168                                 pcpu->target_freq = policy->max;
1169                         else if (policy->min > pcpu->target_freq)
1170                                 pcpu->target_freq = policy->min;
1171
1172                         /* Reschedule timer.
1173                          * Delete the timers, else the timer callback may
1174                          * return without re-arm the timer when failed
1175                          * acquire the semaphore. This race may cause timer
1176                          * stopped unexpectedly.
1177                          */
1178                         del_timer_sync(&pcpu->cpu_timer);
1179                         del_timer_sync(&pcpu->cpu_slack_timer);
1180                         cpufreq_interactive_timer_start(j);
1181                         up_write(&pcpu->enable_sem);
1182                 }
1183                 break;
1184         }
1185         return 0;
1186 }
1187
1188 static void cpufreq_interactive_nop_timer(unsigned long data)
1189 {
1190 }
1191
1192 static int __init cpufreq_interactive_init(void)
1193 {
1194         unsigned int i;
1195         struct cpufreq_interactive_cpuinfo *pcpu;
1196         struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
1197
1198         /* Initalize per-cpu timers */
1199         for_each_possible_cpu(i) {
1200                 pcpu = &per_cpu(cpuinfo, i);
1201                 init_timer_deferrable(&pcpu->cpu_timer);
1202                 pcpu->cpu_timer.function = cpufreq_interactive_timer;
1203                 pcpu->cpu_timer.data = i;
1204                 init_timer(&pcpu->cpu_slack_timer);
1205                 pcpu->cpu_slack_timer.function = cpufreq_interactive_nop_timer;
1206                 spin_lock_init(&pcpu->load_lock);
1207                 init_rwsem(&pcpu->enable_sem);
1208         }
1209
1210         spin_lock_init(&target_loads_lock);
1211         spin_lock_init(&speedchange_cpumask_lock);
1212         spin_lock_init(&above_hispeed_delay_lock);
1213         mutex_init(&gov_lock);
1214         speedchange_task =
1215                 kthread_create(cpufreq_interactive_speedchange_task, NULL,
1216                                "cfinteractive");
1217         if (IS_ERR(speedchange_task))
1218                 return PTR_ERR(speedchange_task);
1219
1220         sched_setscheduler_nocheck(speedchange_task, SCHED_FIFO, &param);
1221         get_task_struct(speedchange_task);
1222
1223         /* NB: wake up so the thread does not look hung to the freezer */
1224         wake_up_process(speedchange_task);
1225
1226         return cpufreq_register_governor(&cpufreq_gov_interactive);
1227 }
1228
1229 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
1230 fs_initcall(cpufreq_interactive_init);
1231 #else
1232 module_init(cpufreq_interactive_init);
1233 #endif
1234
1235 static void __exit cpufreq_interactive_exit(void)
1236 {
1237         cpufreq_unregister_governor(&cpufreq_gov_interactive);
1238         kthread_stop(speedchange_task);
1239         put_task_struct(speedchange_task);
1240 }
1241
1242 module_exit(cpufreq_interactive_exit);
1243
1244 MODULE_AUTHOR("Mike Chan <mike@android.com>");
1245 MODULE_DESCRIPTION("'cpufreq_interactive' - A cpufreq governor for "
1246         "Latency sensitive workloads");
1247 MODULE_LICENSE("GPL");