drivers/cpufreq/cpufreq_interactive.c

   1 /*
   2  * drivers/cpufreq/cpufreq_interactive.c
   3  *
   4  * Copyright (C) 2010 Google, Inc.
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * Author: Mike Chan (mike@android.com)
  16  *
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/cpumask.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/module.h>
  23 #include <linux/mutex.h>
  24 #include <linux/sched.h>
  25 #include <linux/sched/rt.h>
  26 #include <linux/tick.h>
  27 #include <linux/time.h>
  28 #include <linux/timer.h>
  29 #include <linux/workqueue.h>
  30 #include <linux/kthread.h>
  31 #include <linux/mutex.h>
  32 #include <linux/slab.h>
  33 #include <linux/input.h>
  34 #include <asm/cputime.h>
  35
  36 #define CREATE_TRACE_POINTS
  37 #include <trace/events/cpufreq_interactive.h>
  38
  39 static atomic_t active_count = ATOMIC_INIT(0);
  40
  41 struct cpufreq_interactive_cpuinfo {
  42         struct timer_list cpu_timer;
  43         int timer_idlecancel;
  44         u64 time_in_idle;
  45         u64 idle_exit_time;
  46         u64 timer_run_time;
  47         int idling;
  48         u64 target_set_time;
  49         u64 target_set_time_in_idle;
  50         u64 target_validate_time;
  51         struct cpufreq_policy *policy;
  52         struct cpufreq_frequency_table *freq_table;
  53         unsigned int target_freq;
  54         int governor_enabled;
  55 };
  56
  57 static DEFINE_PER_CPU(struct cpufreq_interactive_cpuinfo, cpuinfo);
  58
  59 /* Workqueues handle frequency scaling */
  60 static struct task_struct *up_task;
  61 static struct workqueue_struct *down_wq;
  62 static struct work_struct freq_scale_down_work;
  63 static cpumask_t up_cpumask;
  64 static spinlock_t up_cpumask_lock;
  65 static cpumask_t down_cpumask;
  66 static spinlock_t down_cpumask_lock;
  67 static struct mutex set_speed_lock;
  68
  69 /* Hi speed to bump to from lo speed when load burst (default max) */
  70 static u64 hispeed_freq;
  71
  72 /* Go to hi speed when CPU load at or above this value. */
  73 #define DEFAULT_GO_HISPEED_LOAD 85
  74 static unsigned long go_hispeed_load;
  75
  76 /*
  77  * The minimum amount of time to spend at a frequency before we can ramp down.
  78  */
  79 #define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
  80 static unsigned long min_sample_time;
  81
  82 /*
  83  * The sample rate of the timer used to increase frequency
  84  */
  85 #define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC)
  86 static unsigned long timer_rate;
  87
  88 /*
  89  * Wait this long before raising speed above hispeed, by default a single
  90  * timer interval.
  91  */
  92 #define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE
  93 static unsigned long above_hispeed_delay_val;
  94
  95 /*
  96  * Boost pulse to hispeed on touchscreen input.
  97  */
  98
  99 static int input_boost_val;
 100
 101 struct cpufreq_interactive_inputopen {
 102         struct input_handle *handle;
 103         struct work_struct inputopen_work;
 104 };
 105
 106 static struct cpufreq_interactive_inputopen inputopen;
 107
 108 /*
 109  * Non-zero means longer-term speed boost active.
 110  */
 111
 112 static int boost_val;
 113
 114 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 115                 unsigned int event);
 116
 117 #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 118 static
 119 #endif
 120 struct cpufreq_governor cpufreq_gov_interactive = {
 121         .name = "interactive",
 122         .governor = cpufreq_governor_interactive,
 123         .max_transition_latency = 10000000,
 124         .owner = THIS_MODULE,
 125 };
 126
 127 static void cpufreq_interactive_timer(unsigned long data)
 128 {
 129         unsigned int delta_idle;
 130         unsigned int delta_time;
 131         int cpu_load;
 132         int load_since_change;
 133         u64 time_in_idle;
 134         u64 idle_exit_time;
 135         struct cpufreq_interactive_cpuinfo *pcpu =
 136                 &per_cpu(cpuinfo, data);
 137         u64 now_idle;
 138         unsigned int new_freq;
 139         unsigned int index;
 140         unsigned long flags;
 141
 142         smp_rmb();
 143
 144         if (!pcpu->governor_enabled)
 145                 goto exit;
 146
 147         /*
 148          * Once pcpu->timer_run_time is updated to >= pcpu->idle_exit_time,
 149          * this lets idle exit know the current idle time sample has
 150          * been processed, and idle exit can generate a new sample and
 151          * re-arm the timer.  This prevents a concurrent idle
 152          * exit on that CPU from writing a new set of info at the same time
 153          * the timer function runs (the timer function can't use that info
 154          * until more time passes).
 155          */
 156         time_in_idle = pcpu->time_in_idle;
 157         idle_exit_time = pcpu->idle_exit_time;
 158         now_idle = get_cpu_idle_time_us(data, &pcpu->timer_run_time);
 159         smp_wmb();
 160
 161         /* If we raced with cancelling a timer, skip. */
 162         if (!idle_exit_time)
 163                 goto exit;
 164
 165         delta_idle = (unsigned int)(now_idle - time_in_idle);
 166         delta_time = (unsigned int)(pcpu->timer_run_time - idle_exit_time);
 167
 168         /*
 169          * If timer ran less than 1ms after short-term sample started, retry.
 170          */
 171         if (delta_time < 1000)
 172                 goto rearm;
 173
 174         if (delta_idle > delta_time)
 175                 cpu_load = 0;
 176         else
 177                 cpu_load = 100 * (delta_time - delta_idle) / delta_time;
 178
 179         delta_idle = (unsigned int)(now_idle - pcpu->target_set_time_in_idle);
 180         delta_time = (unsigned int)(pcpu->timer_run_time -
 181                                     pcpu->target_set_time);
 182
 183         if ((delta_time == 0) || (delta_idle > delta_time))
 184                 load_since_change = 0;
 185         else
 186                 load_since_change =
 187                         100 * (delta_time - delta_idle) / delta_time;
 188
 189         /*
 190          * Choose greater of short-term load (since last idle timer
 191          * started or timer function re-armed itself) or long-term load
 192          * (since last frequency change).
 193          */
 194         if (load_since_change > cpu_load)
 195                 cpu_load = load_since_change;
 196
 197         if (cpu_load >= go_hispeed_load || boost_val) {
 198                 if (pcpu->target_freq <= pcpu->policy->min) {
 199                         new_freq = hispeed_freq;
 200                 } else {
 201                         new_freq = pcpu->policy->max * cpu_load / 100;
 202
 203                         if (new_freq < hispeed_freq)
 204                                 new_freq = hispeed_freq;
 205
 206                         if (pcpu->target_freq == hispeed_freq &&
 207                             new_freq > hispeed_freq &&
 208                             pcpu->timer_run_time - pcpu->target_set_time
 209                             < above_hispeed_delay_val) {
 210                                 trace_cpufreq_interactive_notyet(data, cpu_load,
 211                                                                  pcpu->target_freq,
 212                                                                  new_freq);
 213                                 goto rearm;
 214                         }
 215                 }
 216         } else {
 217                 new_freq = pcpu->policy->max * cpu_load / 100;
 218         }
 219
 220         if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
 221                                            new_freq, CPUFREQ_RELATION_H,
 222                                            &index)) {
 223                 pr_warn_once("timer %d: cpufreq_frequency_table_target error\n",
 224                              (int) data);
 225                 goto rearm;
 226         }
 227
 228         new_freq = pcpu->freq_table[index].frequency;
 229
 230         /*
 231          * Do not scale down unless we have been at this frequency for the
 232          * minimum sample time since last validated.
 233          */
 234         if (new_freq < pcpu->target_freq) {
 235                 if (pcpu->timer_run_time - pcpu->target_validate_time
 236                     < min_sample_time) {
 237                         trace_cpufreq_interactive_notyet(data, cpu_load,
 238                                          pcpu->target_freq, new_freq);
 239                         goto rearm;
 240                 }
 241         }
 242
 243         pcpu->target_validate_time = pcpu->timer_run_time;
 244
 245         if (pcpu->target_freq == new_freq) {
 246                 trace_cpufreq_interactive_already(data, cpu_load,
 247                                                   pcpu->target_freq, new_freq);
 248                 goto rearm_if_notmax;
 249         }
 250
 251         trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
 252                                          new_freq);
 253         pcpu->target_set_time_in_idle = now_idle;
 254         pcpu->target_set_time = pcpu->timer_run_time;
 255
 256         if (new_freq < pcpu->target_freq) {
 257                 pcpu->target_freq = new_freq;
 258                 spin_lock_irqsave(&down_cpumask_lock, flags);
 259                 cpumask_set_cpu(data, &down_cpumask);
 260                 spin_unlock_irqrestore(&down_cpumask_lock, flags);
 261                 queue_work(down_wq, &freq_scale_down_work);
 262         } else {
 263                 pcpu->target_freq = new_freq;
 264                 spin_lock_irqsave(&up_cpumask_lock, flags);
 265                 cpumask_set_cpu(data, &up_cpumask);
 266                 spin_unlock_irqrestore(&up_cpumask_lock, flags);
 267                 wake_up_process(up_task);
 268         }
 269
 270 rearm_if_notmax:
 271         /*
 272          * Already set max speed and don't see a need to change that,
 273          * wait until next idle to re-evaluate, don't need timer.
 274          */
 275         if (pcpu->target_freq == pcpu->policy->max)
 276                 goto exit;
 277
 278 rearm:
 279         if (!timer_pending(&pcpu->cpu_timer)) {
 280                 /*
 281                  * If already at min: if that CPU is idle, don't set timer.
 282                  * Else cancel the timer if that CPU goes idle.  We don't
 283                  * need to re-evaluate speed until the next idle exit.
 284                  */
 285                 if (pcpu->target_freq == pcpu->policy->min) {
 286                         smp_rmb();
 287
 288                         if (pcpu->idling)
 289                                 goto exit;
 290
 291                         pcpu->timer_idlecancel = 1;
 292                 }
 293
 294                 pcpu->time_in_idle = get_cpu_idle_time_us(
 295                         data, &pcpu->idle_exit_time);
 296                 mod_timer(&pcpu->cpu_timer,
 297                           jiffies + usecs_to_jiffies(timer_rate));
 298         }
 299
 300 exit:
 301         return;
 302 }
 303
 304 static void cpufreq_interactive_idle_start(void)
 305 {
 306         struct cpufreq_interactive_cpuinfo *pcpu =
 307                 &per_cpu(cpuinfo, smp_processor_id());
 308         int pending;
 309
 310         if (!pcpu->governor_enabled)
 311                 return;
 312
 313         pcpu->idling = 1;
 314         smp_wmb();
 315         pending = timer_pending(&pcpu->cpu_timer);
 316
 317         if (pcpu->target_freq != pcpu->policy->min) {
 318 #ifdef CONFIG_SMP
 319                 /*
 320                  * Entering idle while not at lowest speed.  On some
 321                  * platforms this can hold the other CPU(s) at that speed
 322                  * even though the CPU is idle. Set a timer to re-evaluate
 323                  * speed so this idle CPU doesn't hold the other CPUs above
 324                  * min indefinitely.  This should probably be a quirk of
 325                  * the CPUFreq driver.
 326                  */
 327                 if (!pending) {
 328                         pcpu->time_in_idle = get_cpu_idle_time_us(
 329                                 smp_processor_id(), &pcpu->idle_exit_time);
 330                         pcpu->timer_idlecancel = 0;
 331                         mod_timer(&pcpu->cpu_timer,
 332                                   jiffies + usecs_to_jiffies(timer_rate));
 333                 }
 334 #endif
 335         } else {
 336                 /*
 337                  * If at min speed and entering idle after load has
 338                  * already been evaluated, and a timer has been set just in
 339                  * case the CPU suddenly goes busy, cancel that timer.  The
 340                  * CPU didn't go busy; we'll recheck things upon idle exit.
 341                  */
 342                 if (pending && pcpu->timer_idlecancel) {
 343                         del_timer(&pcpu->cpu_timer);
 344                         /*
 345                          * Ensure last timer run time is after current idle
 346                          * sample start time, so next idle exit will always
 347                          * start a new idle sampling period.
 348                          */
 349                         pcpu->idle_exit_time = 0;
 350                         pcpu->timer_idlecancel = 0;
 351                 }
 352         }
 353
 354 }
 355
 356 static void cpufreq_interactive_idle_end(void)
 357 {
 358         struct cpufreq_interactive_cpuinfo *pcpu =
 359                 &per_cpu(cpuinfo, smp_processor_id());
 360
 361         pcpu->idling = 0;
 362         smp_wmb();
 363
 364         /*
 365          * Arm the timer for 1-2 ticks later if not already, and if the timer
 366          * function has already processed the previous load sampling
 367          * interval.  (If the timer is not pending but has not processed
 368          * the previous interval, it is probably racing with us on another
 369          * CPU.  Let it compute load based on the previous sample and then
 370          * re-arm the timer for another interval when it's done, rather
 371          * than updating the interval start time to be "now", which doesn't
 372          * give the timer function enough time to make a decision on this
 373          * run.)
 374          */
 375         if (timer_pending(&pcpu->cpu_timer) == 0 &&
 376             pcpu->timer_run_time >= pcpu->idle_exit_time &&
 377             pcpu->governor_enabled) {
 378                 pcpu->time_in_idle =
 379                         get_cpu_idle_time_us(smp_processor_id(),
 380                                              &pcpu->idle_exit_time);
 381                 pcpu->timer_idlecancel = 0;
 382                 mod_timer(&pcpu->cpu_timer,
 383                           jiffies + usecs_to_jiffies(timer_rate));
 384         }
 385
 386 }
 387
 388 static int cpufreq_interactive_up_task(void *data)
 389 {
 390         unsigned int cpu;
 391         cpumask_t tmp_mask;
 392         unsigned long flags;
 393         struct cpufreq_interactive_cpuinfo *pcpu;
 394
 395         while (1) {
 396                 set_current_state(TASK_INTERRUPTIBLE);
 397                 spin_lock_irqsave(&up_cpumask_lock, flags);
 398
 399                 if (cpumask_empty(&up_cpumask)) {
 400                         spin_unlock_irqrestore(&up_cpumask_lock, flags);
 401                         schedule();
 402
 403                         if (kthread_should_stop())
 404                                 break;
 405
 406                         spin_lock_irqsave(&up_cpumask_lock, flags);
 407                 }
 408
 409                 set_current_state(TASK_RUNNING);
 410                 tmp_mask = up_cpumask;
 411                 cpumask_clear(&up_cpumask);
 412                 spin_unlock_irqrestore(&up_cpumask_lock, flags);
 413
 414                 for_each_cpu(cpu, &tmp_mask) {
 415                         unsigned int j;
 416                         unsigned int max_freq = 0;
 417
 418                         pcpu = &per_cpu(cpuinfo, cpu);
 419                         smp_rmb();
 420
 421                         if (!pcpu->governor_enabled)
 422                                 continue;
 423
 424                         mutex_lock(&set_speed_lock);
 425
 426                         for_each_cpu(j, pcpu->policy->cpus) {
 427                                 struct cpufreq_interactive_cpuinfo *pjcpu =
 428                                         &per_cpu(cpuinfo, j);
 429
 430                                 if (pjcpu->target_freq > max_freq)
 431                                         max_freq = pjcpu->target_freq;
 432                         }
 433
 434                         if (max_freq != pcpu->policy->cur)
 435                                 __cpufreq_driver_target(pcpu->policy,
 436                                                         max_freq,
 437                                                         CPUFREQ_RELATION_H);
 438                         mutex_unlock(&set_speed_lock);
 439                         trace_cpufreq_interactive_up(cpu, pcpu->target_freq,
 440                                                      pcpu->policy->cur);
 441                 }
 442         }
 443
 444         return 0;
 445 }
 446
 447 static void cpufreq_interactive_freq_down(struct work_struct *work)
 448 {
 449         unsigned int cpu;
 450         cpumask_t tmp_mask;
 451         unsigned long flags;
 452         struct cpufreq_interactive_cpuinfo *pcpu;
 453
 454         spin_lock_irqsave(&down_cpumask_lock, flags);
 455         tmp_mask = down_cpumask;
 456         cpumask_clear(&down_cpumask);
 457         spin_unlock_irqrestore(&down_cpumask_lock, flags);
 458
 459         for_each_cpu(cpu, &tmp_mask) {
 460                 unsigned int j;
 461                 unsigned int max_freq = 0;
 462
 463                 pcpu = &per_cpu(cpuinfo, cpu);
 464                 smp_rmb();
 465
 466                 if (!pcpu->governor_enabled)
 467                         continue;
 468
 469                 mutex_lock(&set_speed_lock);
 470
 471                 for_each_cpu(j, pcpu->policy->cpus) {
 472                         struct cpufreq_interactive_cpuinfo *pjcpu =
 473                                 &per_cpu(cpuinfo, j);
 474
 475                         if (pjcpu->target_freq > max_freq)
 476                                 max_freq = pjcpu->target_freq;
 477                 }
 478
 479                 if (max_freq != pcpu->policy->cur)
 480                         __cpufreq_driver_target(pcpu->policy, max_freq,
 481                                                 CPUFREQ_RELATION_H);
 482
 483                 mutex_unlock(&set_speed_lock);
 484                 trace_cpufreq_interactive_down(cpu, pcpu->target_freq,
 485                                                pcpu->policy->cur);
 486         }
 487 }
 488
 489 static void cpufreq_interactive_boost(void)
 490 {
 491         int i;
 492         int anyboost = 0;
 493         unsigned long flags;
 494         struct cpufreq_interactive_cpuinfo *pcpu;
 495
 496         trace_cpufreq_interactive_boost(hispeed_freq);
 497         spin_lock_irqsave(&up_cpumask_lock, flags);
 498
 499         for_each_online_cpu(i) {
 500                 pcpu = &per_cpu(cpuinfo, i);
 501
 502                 if (pcpu->target_freq < hispeed_freq) {
 503                         pcpu->target_freq = hispeed_freq;
 504                         cpumask_set_cpu(i, &up_cpumask);
 505                         pcpu->target_set_time_in_idle =
 506                                 get_cpu_idle_time_us(i, &pcpu->target_set_time);
 507                         anyboost = 1;
 508                 }
 509
 510                 /*
 511                  * Refresh time at which current (possibly being
 512                  * boosted) speed last validated (reset timer for
 513                  * allowing speed to drop).
 514                  */
 515
 516                 pcpu->target_validate_time = ktime_to_us(ktime_get());
 517         }
 518
 519         spin_unlock_irqrestore(&up_cpumask_lock, flags);
 520
 521         if (anyboost)
 522                 wake_up_process(up_task);
 523 }
 524
 525 /*
 526  * Pulsed boost on input event raises CPUs to hispeed_freq and lets
 527  * usual algorithm of min_sample_time  decide when to allow speed
 528  * to drop.
 529  */
 530
 531 static void cpufreq_interactive_input_event(struct input_handle *handle,
 532                                             unsigned int type,
 533                                             unsigned int code, int value)
 534 {
 535         if (input_boost_val && type == EV_SYN && code == SYN_REPORT)
 536                 cpufreq_interactive_boost();
 537 }
 538
 539 static void cpufreq_interactive_input_open(struct work_struct *w)
 540 {
 541         struct cpufreq_interactive_inputopen *io =
 542                 container_of(w, struct cpufreq_interactive_inputopen,
 543                              inputopen_work);
 544         int error;
 545
 546         error = input_open_device(io->handle);
 547         if (error)
 548                 input_unregister_handle(io->handle);
 549 }
 550
 551 static int cpufreq_interactive_input_connect(struct input_handler *handler,
 552                                              struct input_dev *dev,
 553                                              const struct input_device_id *id)
 554 {
 555         struct input_handle *handle;
 556         int error;
 557
 558         pr_info("%s: connect to %s\n", __func__, dev->name);
 559         handle = kzalloc(sizeof(struct input_handle), GFP_KERNEL);
 560         if (!handle)
 561                 return -ENOMEM;
 562
 563         handle->dev = dev;
 564         handle->handler = handler;
 565         handle->name = "cpufreq_interactive";
 566
 567         error = input_register_handle(handle);
 568         if (error)
 569                 goto err;
 570
 571         inputopen.handle = handle;
 572         queue_work(down_wq, &inputopen.inputopen_work);
 573         return 0;
 574 err:
 575         kfree(handle);
 576         return error;
 577 }
 578
 579 static void cpufreq_interactive_input_disconnect(struct input_handle *handle)
 580 {
 581         input_close_device(handle);
 582         input_unregister_handle(handle);
 583         kfree(handle);
 584 }
 585
 586 static const struct input_device_id cpufreq_interactive_ids[] = {
 587         {
 588                 .flags = INPUT_DEVICE_ID_MATCH_EVBIT |
 589                          INPUT_DEVICE_ID_MATCH_ABSBIT,
 590                 .evbit = { BIT_MASK(EV_ABS) },
 591                 .absbit = { [BIT_WORD(ABS_MT_POSITION_X)] =
 592                             BIT_MASK(ABS_MT_POSITION_X) |
 593                             BIT_MASK(ABS_MT_POSITION_Y) },
 594         }, /* multi-touch touchscreen */
 595         {
 596                 .flags = INPUT_DEVICE_ID_MATCH_KEYBIT |
 597                          INPUT_DEVICE_ID_MATCH_ABSBIT,
 598                 .keybit = { [BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH) },
 599                 .absbit = { [BIT_WORD(ABS_X)] =
 600                             BIT_MASK(ABS_X) | BIT_MASK(ABS_Y) },
 601         }, /* touchpad */
 602         { },
 603 };
 604
 605 static struct input_handler cpufreq_interactive_input_handler = {
 606         .event          = cpufreq_interactive_input_event,
 607         .connect        = cpufreq_interactive_input_connect,
 608         .disconnect     = cpufreq_interactive_input_disconnect,
 609         .name           = "cpufreq_interactive",
 610         .id_table       = cpufreq_interactive_ids,
 611 };
 612
 613 static ssize_t show_hispeed_freq(struct kobject *kobj,
 614                                  struct attribute *attr, char *buf)
 615 {
 616         return sprintf(buf, "%llu\n", hispeed_freq);
 617 }
 618
 619 static ssize_t store_hispeed_freq(struct kobject *kobj,
 620                                   struct attribute *attr, const char *buf,
 621                                   size_t count)
 622 {
 623         int ret;
 624         u64 val;
 625
 626         ret = strict_strtoull(buf, 0, &val);
 627         if (ret < 0)
 628                 return ret;
 629         hispeed_freq = val;
 630         return count;
 631 }
 632
 633 static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644,
 634                 show_hispeed_freq, store_hispeed_freq);
 635
 636
 637 static ssize_t show_go_hispeed_load(struct kobject *kobj,
 638                                      struct attribute *attr, char *buf)
 639 {
 640         return sprintf(buf, "%lu\n", go_hispeed_load);
 641 }
 642
 643 static ssize_t store_go_hispeed_load(struct kobject *kobj,
 644                         struct attribute *attr, const char *buf, size_t count)
 645 {
 646         int ret;
 647         unsigned long val;
 648
 649         ret = strict_strtoul(buf, 0, &val);
 650         if (ret < 0)
 651                 return ret;
 652         go_hispeed_load = val;
 653         return count;
 654 }
 655
 656 static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644,
 657                 show_go_hispeed_load, store_go_hispeed_load);
 658
 659 static ssize_t show_min_sample_time(struct kobject *kobj,
 660                                 struct attribute *attr, char *buf)
 661 {
 662         return sprintf(buf, "%lu\n", min_sample_time);
 663 }
 664
 665 static ssize_t store_min_sample_time(struct kobject *kobj,
 666                         struct attribute *attr, const char *buf, size_t count)
 667 {
 668         int ret;
 669         unsigned long val;
 670
 671         ret = strict_strtoul(buf, 0, &val);
 672         if (ret < 0)
 673                 return ret;
 674         min_sample_time = val;
 675         return count;
 676 }
 677
 678 static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
 679                 show_min_sample_time, store_min_sample_time);
 680
 681 static ssize_t show_above_hispeed_delay(struct kobject *kobj,
 682                                         struct attribute *attr, char *buf)
 683 {
 684         return sprintf(buf, "%lu\n", above_hispeed_delay_val);
 685 }
 686
 687 static ssize_t store_above_hispeed_delay(struct kobject *kobj,
 688                                          struct attribute *attr,
 689                                          const char *buf, size_t count)
 690 {
 691         int ret;
 692         unsigned long val;
 693
 694         ret = strict_strtoul(buf, 0, &val);
 695         if (ret < 0)
 696                 return ret;
 697         above_hispeed_delay_val = val;
 698         return count;
 699 }
 700
 701 define_one_global_rw(above_hispeed_delay);
 702
 703 static ssize_t show_timer_rate(struct kobject *kobj,
 704                         struct attribute *attr, char *buf)
 705 {
 706         return sprintf(buf, "%lu\n", timer_rate);
 707 }
 708
 709 static ssize_t store_timer_rate(struct kobject *kobj,
 710                         struct attribute *attr, const char *buf, size_t count)
 711 {
 712         int ret;
 713         unsigned long val;
 714
 715         ret = strict_strtoul(buf, 0, &val);
 716         if (ret < 0)
 717                 return ret;
 718         timer_rate = val;
 719         return count;
 720 }
 721
 722 static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
 723                 show_timer_rate, store_timer_rate);
 724
 725 static ssize_t show_input_boost(struct kobject *kobj, struct attribute *attr,
 726                                 char *buf)
 727 {
 728         return sprintf(buf, "%u\n", input_boost_val);
 729 }
 730
 731 static ssize_t store_input_boost(struct kobject *kobj, struct attribute *attr,
 732                                  const char *buf, size_t count)
 733 {
 734         int ret;
 735         unsigned long val;
 736
 737         ret = strict_strtoul(buf, 0, &val);
 738         if (ret < 0)
 739                 return ret;
 740         input_boost_val = val;
 741         return count;
 742 }
 743
 744 define_one_global_rw(input_boost);
 745
 746 static ssize_t show_boost(struct kobject *kobj, struct attribute *attr,
 747                           char *buf)
 748 {
 749         return sprintf(buf, "%d\n", boost_val);
 750 }
 751
 752 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 753                            const char *buf, size_t count)
 754 {
 755         int ret;
 756         unsigned long val;
 757
 758         ret = kstrtoul(buf, 0, &val);
 759         if (ret < 0)
 760                 return ret;
 761
 762         boost_val = val;
 763
 764         if (boost_val)
 765                 cpufreq_interactive_boost();
 766         else
 767                 trace_cpufreq_interactive_unboost(hispeed_freq);
 768
 769         return count;
 770 }
 771
 772 define_one_global_rw(boost);
 773
 774 static struct attribute *interactive_attributes[] = {
 775         &hispeed_freq_attr.attr,
 776         &go_hispeed_load_attr.attr,
 777         &above_hispeed_delay.attr,
 778         &min_sample_time_attr.attr,
 779         &timer_rate_attr.attr,
 780         &input_boost.attr,
 781         &boost.attr,
 782         NULL,
 783 };
 784
 785 static struct attribute_group interactive_attr_group = {
 786         .attrs = interactive_attributes,
 787         .name = "interactive",
 788 };
 789
 790 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 791                 unsigned int event)
 792 {
 793         int rc;
 794         unsigned int j;
 795         struct cpufreq_interactive_cpuinfo *pcpu;
 796         struct cpufreq_frequency_table *freq_table;
 797
 798         switch (event) {
 799         case CPUFREQ_GOV_START:
 800                 if (!cpu_online(policy->cpu))
 801                         return -EINVAL;
 802
 803                 freq_table =
 804                         cpufreq_frequency_get_table(policy->cpu);
 805
 806                 for_each_cpu(j, policy->cpus) {
 807                         pcpu = &per_cpu(cpuinfo, j);
 808                         pcpu->policy = policy;
 809                         pcpu->target_freq = policy->cur;
 810                         pcpu->freq_table = freq_table;
 811                         pcpu->target_set_time_in_idle =
 812                                 get_cpu_idle_time_us(j,
 813                                              &pcpu->target_set_time);
 814                         pcpu->target_validate_time =
 815                                 pcpu->target_set_time;
 816                         pcpu->governor_enabled = 1;
 817                         smp_wmb();
 818                 }
 819
 820                 if (!hispeed_freq)
 821                         hispeed_freq = policy->max;
 822
 823                 /*
 824                  * Do not register the idle hook and create sysfs
 825                  * entries if we have already done so.
 826                  */
 827                 if (atomic_inc_return(&active_count) > 1)
 828                         return 0;
 829
 830                 rc = sysfs_create_group(cpufreq_global_kobject,
 831                                 &interactive_attr_group);
 832                 if (rc)
 833                         return rc;
 834
 835                 rc = input_register_handler(&cpufreq_interactive_input_handler);
 836                 if (rc)
 837                         pr_warn("%s: failed to register input handler\n",
 838                                 __func__);
 839
 840                 break;
 841
 842         case CPUFREQ_GOV_STOP:
 843                 for_each_cpu(j, policy->cpus) {
 844                         pcpu = &per_cpu(cpuinfo, j);
 845                         pcpu->governor_enabled = 0;
 846                         smp_wmb();
 847                         del_timer_sync(&pcpu->cpu_timer);
 848
 849                         /*
 850                          * Reset idle exit time since we may cancel the timer
 851                          * before it can run after the last idle exit time,
 852                          * to avoid tripping the check in idle exit for a timer
 853                          * that is trying to run.
 854                          */
 855                         pcpu->idle_exit_time = 0;
 856                 }
 857
 858                 flush_work(&freq_scale_down_work);
 859                 if (atomic_dec_return(&active_count) > 0)
 860                         return 0;
 861
 862                 input_unregister_handler(&cpufreq_interactive_input_handler);
 863                 sysfs_remove_group(cpufreq_global_kobject,
 864                                 &interactive_attr_group);
 865
 866                 break;
 867
 868         case CPUFREQ_GOV_LIMITS:
 869                 if (policy->max < policy->cur)
 870                         __cpufreq_driver_target(policy,
 871                                         policy->max, CPUFREQ_RELATION_H);
 872                 else if (policy->min > policy->cur)
 873                         __cpufreq_driver_target(policy,
 874                                         policy->min, CPUFREQ_RELATION_L);
 875                 break;
 876         }
 877         return 0;
 878 }
 879
 880 static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
 881                                              unsigned long val,
 882                                              void *data)
 883 {
 884         switch (val) {
 885         case IDLE_START:
 886                 cpufreq_interactive_idle_start();
 887                 break;
 888         case IDLE_END:
 889                 cpufreq_interactive_idle_end();
 890                 break;
 891         }
 892
 893         return 0;
 894 }
 895
 896 static struct notifier_block cpufreq_interactive_idle_nb = {
 897         .notifier_call = cpufreq_interactive_idle_notifier,
 898 };
 899
 900 static int __init cpufreq_interactive_init(void)
 901 {
 902         unsigned int i;
 903         struct cpufreq_interactive_cpuinfo *pcpu;
 904         struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 905
 906         go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
 907         min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
 908         above_hispeed_delay_val = DEFAULT_ABOVE_HISPEED_DELAY;
 909         timer_rate = DEFAULT_TIMER_RATE;
 910
 911         /* Initalize per-cpu timers */
 912         for_each_possible_cpu(i) {
 913                 pcpu = &per_cpu(cpuinfo, i);
 914                 init_timer(&pcpu->cpu_timer);
 915                 pcpu->cpu_timer.function = cpufreq_interactive_timer;
 916                 pcpu->cpu_timer.data = i;
 917         }
 918
 919         up_task = kthread_create(cpufreq_interactive_up_task, NULL,
 920                                  "kinteractiveup");
 921         if (IS_ERR(up_task))
 922                 return PTR_ERR(up_task);
 923
 924         sched_setscheduler_nocheck(up_task, SCHED_FIFO, &param);
 925         get_task_struct(up_task);
 926
 927         /* No rescuer thread, bind to CPU queuing the work for possibly
 928            warm cache (probably doesn't matter much). */
 929         down_wq = alloc_workqueue("knteractive_down", 0, 1);
 930
 931         if (!down_wq)
 932                 goto err_freeuptask;
 933
 934         INIT_WORK(&freq_scale_down_work,
 935                   cpufreq_interactive_freq_down);
 936
 937         spin_lock_init(&up_cpumask_lock);
 938         spin_lock_init(&down_cpumask_lock);
 939         mutex_init(&set_speed_lock);
 940
 941         idle_notifier_register(&cpufreq_interactive_idle_nb);
 942         INIT_WORK(&inputopen.inputopen_work, cpufreq_interactive_input_open);
 943         return cpufreq_register_governor(&cpufreq_gov_interactive);
 944
 945 err_freeuptask:
 946         put_task_struct(up_task);
 947         return -ENOMEM;
 948 }
 949
 950 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 951 fs_initcall(cpufreq_interactive_init);
 952 #else
 953 module_init(cpufreq_interactive_init);
 954 #endif
 955
 956 static void __exit cpufreq_interactive_exit(void)
 957 {
 958         cpufreq_unregister_governor(&cpufreq_gov_interactive);
 959         kthread_stop(up_task);
 960         put_task_struct(up_task);
 961         destroy_workqueue(down_wq);
 962 }
 963
 964 module_exit(cpufreq_interactive_exit);
 965
 966 MODULE_AUTHOR("Mike Chan <mike@android.com>");
 967 MODULE_DESCRIPTION("'cpufreq_interactive' - A cpufreq governor for "
 968         "Latency sensitive workloads");
 969 MODULE_LICENSE("GPL");