drivers/cpufreq/cpufreq_interactive.c

   1 /*
   2  * drivers/cpufreq/cpufreq_interactive.c
   3  *
   4  * Copyright (C) 2010 Google, Inc.
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * Author: Mike Chan (mike@android.com)
  16  *
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/cpumask.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/module.h>
  23 #include <linux/mutex.h>
  24 #include <linux/sched.h>
  25 #include <linux/sched/rt.h>
  26 #include <linux/tick.h>
  27 #include <linux/time.h>
  28 #include <linux/timer.h>
  29 #include <linux/workqueue.h>
  30 #include <linux/kthread.h>
  31 #include <linux/mutex.h>
  32 #include <linux/slab.h>
  33 #include <linux/input.h>
  34 #include <asm/cputime.h>
  35
  36 #define CREATE_TRACE_POINTS
  37 #include <trace/events/cpufreq_interactive.h>
  38
  39 static atomic_t active_count = ATOMIC_INIT(0);
  40
  41 struct cpufreq_interactive_cpuinfo {
  42         struct timer_list cpu_timer;
  43         int timer_idlecancel;
  44         u64 time_in_idle;
  45         u64 idle_exit_time;
  46         u64 timer_run_time;
  47         int idling;
  48         u64 target_set_time;
  49         u64 target_set_time_in_idle;
  50         struct cpufreq_policy *policy;
  51         struct cpufreq_frequency_table *freq_table;
  52         unsigned int target_freq;
  53         unsigned int floor_freq;
  54         u64 floor_validate_time;
  55         u64 hispeed_validate_time;
  56         int governor_enabled;
  57 };
  58
  59 static DEFINE_PER_CPU(struct cpufreq_interactive_cpuinfo, cpuinfo);
  60
  61 /* realtime thread handles frequency scaling */
  62 static struct task_struct *speedchange_task;
  63 static cpumask_t speedchange_cpumask;
  64 static spinlock_t speedchange_cpumask_lock;
  65
  66 /* Hi speed to bump to from lo speed when load burst (default max) */
  67 static u64 hispeed_freq;
  68
  69 /* Go to hi speed when CPU load at or above this value. */
  70 #define DEFAULT_GO_HISPEED_LOAD 85
  71 static unsigned long go_hispeed_load;
  72
  73 /*
  74  * The minimum amount of time to spend at a frequency before we can ramp down.
  75  */
  76 #define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
  77 static unsigned long min_sample_time;
  78
  79 /*
  80  * The sample rate of the timer used to increase frequency
  81  */
  82 #define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC)
  83 static unsigned long timer_rate;
  84
  85 /*
  86  * Wait this long before raising speed above hispeed, by default a single
  87  * timer interval.
  88  */
  89 #define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE
  90 static unsigned long above_hispeed_delay_val;
  91
  92 /*
  93  * Boost pulse to hispeed on touchscreen input.
  94  */
  95
  96 static int input_boost_val;
  97
  98 struct cpufreq_interactive_inputopen {
  99         struct input_handle *handle;
 100         struct work_struct inputopen_work;
 101 };
 102
 103 static struct cpufreq_interactive_inputopen inputopen;
 104 static struct workqueue_struct *inputopen_wq;
 105
 106 /*
 107  * Non-zero means longer-term speed boost active.
 108  */
 109
 110 static int boost_val;
 111
 112 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 113                 unsigned int event);
 114
 115 #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 116 static
 117 #endif
 118 struct cpufreq_governor cpufreq_gov_interactive = {
 119         .name = "interactive",
 120         .governor = cpufreq_governor_interactive,
 121         .max_transition_latency = 10000000,
 122         .owner = THIS_MODULE,
 123 };
 124
 125 static void cpufreq_interactive_timer(unsigned long data)
 126 {
 127         unsigned int delta_idle;
 128         unsigned int delta_time;
 129         int cpu_load;
 130         int load_since_change;
 131         u64 time_in_idle;
 132         u64 idle_exit_time;
 133         struct cpufreq_interactive_cpuinfo *pcpu =
 134                 &per_cpu(cpuinfo, data);
 135         u64 now_idle;
 136         unsigned int new_freq;
 137         unsigned int index;
 138         unsigned long flags;
 139
 140         smp_rmb();
 141
 142         if (!pcpu->governor_enabled)
 143                 goto exit;
 144
 145         /*
 146          * Once pcpu->timer_run_time is updated to >= pcpu->idle_exit_time,
 147          * this lets idle exit know the current idle time sample has
 148          * been processed, and idle exit can generate a new sample and
 149          * re-arm the timer.  This prevents a concurrent idle
 150          * exit on that CPU from writing a new set of info at the same time
 151          * the timer function runs (the timer function can't use that info
 152          * until more time passes).
 153          */
 154         time_in_idle = pcpu->time_in_idle;
 155         idle_exit_time = pcpu->idle_exit_time;
 156         now_idle = get_cpu_idle_time_us(data, &pcpu->timer_run_time);
 157         smp_wmb();
 158
 159         /* If we raced with cancelling a timer, skip. */
 160         if (!idle_exit_time)
 161                 goto exit;
 162
 163         delta_idle = (unsigned int)(now_idle - time_in_idle);
 164         delta_time = (unsigned int)(pcpu->timer_run_time - idle_exit_time);
 165
 166         /*
 167          * If timer ran less than 1ms after short-term sample started, retry.
 168          */
 169         if (delta_time < 1000)
 170                 goto rearm;
 171
 172         if (delta_idle > delta_time)
 173                 cpu_load = 0;
 174         else
 175                 cpu_load = 100 * (delta_time - delta_idle) / delta_time;
 176
 177         delta_idle = (unsigned int)(now_idle - pcpu->target_set_time_in_idle);
 178         delta_time = (unsigned int)(pcpu->timer_run_time -
 179                                     pcpu->target_set_time);
 180
 181         if ((delta_time == 0) || (delta_idle > delta_time))
 182                 load_since_change = 0;
 183         else
 184                 load_since_change =
 185                         100 * (delta_time - delta_idle) / delta_time;
 186
 187         /*
 188          * Choose greater of short-term load (since last idle timer
 189          * started or timer function re-armed itself) or long-term load
 190          * (since last frequency change).
 191          */
 192         if (load_since_change > cpu_load)
 193                 cpu_load = load_since_change;
 194
 195         if (cpu_load >= go_hispeed_load || boost_val) {
 196                 if (pcpu->target_freq <= pcpu->policy->min) {
 197                         new_freq = hispeed_freq;
 198                 } else {
 199                         new_freq = pcpu->policy->max * cpu_load / 100;
 200
 201                         if (new_freq < hispeed_freq)
 202                                 new_freq = hispeed_freq;
 203
 204                         if (pcpu->target_freq == hispeed_freq &&
 205                             new_freq > hispeed_freq &&
 206                             pcpu->timer_run_time - pcpu->hispeed_validate_time
 207                             < above_hispeed_delay_val) {
 208                                 trace_cpufreq_interactive_notyet(data, cpu_load,
 209                                                                  pcpu->target_freq,
 210                                                                  new_freq);
 211                                 goto rearm;
 212                         }
 213                 }
 214         } else {
 215                 new_freq = pcpu->policy->max * cpu_load / 100;
 216         }
 217
 218         if (new_freq <= hispeed_freq)
 219                 pcpu->hispeed_validate_time = pcpu->timer_run_time;
 220
 221         if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
 222                                            new_freq, CPUFREQ_RELATION_H,
 223                                            &index)) {
 224                 pr_warn_once("timer %d: cpufreq_frequency_table_target error\n",
 225                              (int) data);
 226                 goto rearm;
 227         }
 228
 229         new_freq = pcpu->freq_table[index].frequency;
 230
 231         /*
 232          * Do not scale below floor_freq unless we have been at or above the
 233          * floor frequency for the minimum sample time since last validated.
 234          */
 235         if (new_freq < pcpu->floor_freq) {
 236                 if (pcpu->timer_run_time - pcpu->floor_validate_time
 237                     < min_sample_time) {
 238                         trace_cpufreq_interactive_notyet(data, cpu_load,
 239                                          pcpu->target_freq, new_freq);
 240                         goto rearm;
 241                 }
 242         }
 243
 244         pcpu->floor_freq = new_freq;
 245         pcpu->floor_validate_time = pcpu->timer_run_time;
 246
 247         if (pcpu->target_freq == new_freq) {
 248                 trace_cpufreq_interactive_already(data, cpu_load,
 249                                                   pcpu->target_freq, new_freq);
 250                 goto rearm_if_notmax;
 251         }
 252
 253         trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
 254                                          new_freq);
 255         pcpu->target_set_time_in_idle = now_idle;
 256         pcpu->target_set_time = pcpu->timer_run_time;
 257
 258         pcpu->target_freq = new_freq;
 259         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 260         cpumask_set_cpu(data, &speedchange_cpumask);
 261         spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 262         wake_up_process(speedchange_task);
 263
 264 rearm_if_notmax:
 265         /*
 266          * Already set max speed and don't see a need to change that,
 267          * wait until next idle to re-evaluate, don't need timer.
 268          */
 269         if (pcpu->target_freq == pcpu->policy->max)
 270                 goto exit;
 271
 272 rearm:
 273         if (!timer_pending(&pcpu->cpu_timer)) {
 274                 /*
 275                  * If already at min: if that CPU is idle, don't set timer.
 276                  * Else cancel the timer if that CPU goes idle.  We don't
 277                  * need to re-evaluate speed until the next idle exit.
 278                  */
 279                 if (pcpu->target_freq == pcpu->policy->min) {
 280                         smp_rmb();
 281
 282                         if (pcpu->idling)
 283                                 goto exit;
 284
 285                         pcpu->timer_idlecancel = 1;
 286                 }
 287
 288                 pcpu->time_in_idle = get_cpu_idle_time_us(
 289                         data, &pcpu->idle_exit_time);
 290                 mod_timer(&pcpu->cpu_timer,
 291                           jiffies + usecs_to_jiffies(timer_rate));
 292         }
 293
 294 exit:
 295         return;
 296 }
 297
 298 static void cpufreq_interactive_idle_start(void)
 299 {
 300         struct cpufreq_interactive_cpuinfo *pcpu =
 301                 &per_cpu(cpuinfo, smp_processor_id());
 302         int pending;
 303
 304         if (!pcpu->governor_enabled)
 305                 return;
 306
 307         pcpu->idling = 1;
 308         smp_wmb();
 309         pending = timer_pending(&pcpu->cpu_timer);
 310
 311         if (pcpu->target_freq != pcpu->policy->min) {
 312 #ifdef CONFIG_SMP
 313                 /*
 314                  * Entering idle while not at lowest speed.  On some
 315                  * platforms this can hold the other CPU(s) at that speed
 316                  * even though the CPU is idle. Set a timer to re-evaluate
 317                  * speed so this idle CPU doesn't hold the other CPUs above
 318                  * min indefinitely.  This should probably be a quirk of
 319                  * the CPUFreq driver.
 320                  */
 321                 if (!pending) {
 322                         pcpu->time_in_idle = get_cpu_idle_time_us(
 323                                 smp_processor_id(), &pcpu->idle_exit_time);
 324                         pcpu->timer_idlecancel = 0;
 325                         mod_timer(&pcpu->cpu_timer,
 326                                   jiffies + usecs_to_jiffies(timer_rate));
 327                 }
 328 #endif
 329         } else {
 330                 /*
 331                  * If at min speed and entering idle after load has
 332                  * already been evaluated, and a timer has been set just in
 333                  * case the CPU suddenly goes busy, cancel that timer.  The
 334                  * CPU didn't go busy; we'll recheck things upon idle exit.
 335                  */
 336                 if (pending && pcpu->timer_idlecancel) {
 337                         del_timer(&pcpu->cpu_timer);
 338                         /*
 339                          * Ensure last timer run time is after current idle
 340                          * sample start time, so next idle exit will always
 341                          * start a new idle sampling period.
 342                          */
 343                         pcpu->idle_exit_time = 0;
 344                         pcpu->timer_idlecancel = 0;
 345                 }
 346         }
 347
 348 }
 349
 350 static void cpufreq_interactive_idle_end(void)
 351 {
 352         struct cpufreq_interactive_cpuinfo *pcpu =
 353                 &per_cpu(cpuinfo, smp_processor_id());
 354
 355         if (!pcpu->governor_enabled)
 356                 return;
 357
 358         pcpu->idling = 0;
 359         smp_wmb();
 360
 361         /*
 362          * Arm the timer for 1-2 ticks later if not already, and if the timer
 363          * function has already processed the previous load sampling
 364          * interval.  (If the timer is not pending but has not processed
 365          * the previous interval, it is probably racing with us on another
 366          * CPU.  Let it compute load based on the previous sample and then
 367          * re-arm the timer for another interval when it's done, rather
 368          * than updating the interval start time to be "now", which doesn't
 369          * give the timer function enough time to make a decision on this
 370          * run.)
 371          */
 372         if (timer_pending(&pcpu->cpu_timer) == 0 &&
 373             pcpu->timer_run_time >= pcpu->idle_exit_time &&
 374             pcpu->governor_enabled) {
 375                 pcpu->time_in_idle =
 376                         get_cpu_idle_time_us(smp_processor_id(),
 377                                              &pcpu->idle_exit_time);
 378                 pcpu->timer_idlecancel = 0;
 379                 mod_timer(&pcpu->cpu_timer,
 380                           jiffies + usecs_to_jiffies(timer_rate));
 381         }
 382
 383 }
 384
 385 static int cpufreq_interactive_speedchange_task(void *data)
 386 {
 387         unsigned int cpu;
 388         cpumask_t tmp_mask;
 389         unsigned long flags;
 390         struct cpufreq_interactive_cpuinfo *pcpu;
 391
 392         while (1) {
 393                 set_current_state(TASK_INTERRUPTIBLE);
 394                 spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 395
 396                 if (cpumask_empty(&speedchange_cpumask)) {
 397                         spin_unlock_irqrestore(&speedchange_cpumask_lock,
 398                                                flags);
 399                         schedule();
 400
 401                         if (kthread_should_stop())
 402                                 break;
 403
 404                         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 405                 }
 406
 407                 set_current_state(TASK_RUNNING);
 408                 tmp_mask = speedchange_cpumask;
 409                 cpumask_clear(&speedchange_cpumask);
 410                 spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 411
 412                 for_each_cpu(cpu, &tmp_mask) {
 413                         unsigned int j;
 414                         unsigned int max_freq = 0;
 415
 416                         pcpu = &per_cpu(cpuinfo, cpu);
 417                         smp_rmb();
 418
 419                         if (!pcpu->governor_enabled)
 420                                 continue;
 421
 422                         for_each_cpu(j, pcpu->policy->cpus) {
 423                                 struct cpufreq_interactive_cpuinfo *pjcpu =
 424                                         &per_cpu(cpuinfo, j);
 425
 426                                 if (pjcpu->target_freq > max_freq)
 427                                         max_freq = pjcpu->target_freq;
 428                         }
 429
 430                         if (max_freq != pcpu->policy->cur)
 431                                 __cpufreq_driver_target(pcpu->policy,
 432                                                         max_freq,
 433                                                         CPUFREQ_RELATION_H);
 434                         trace_cpufreq_interactive_setspeed(cpu,
 435                                                      pcpu->target_freq,
 436                                                      pcpu->policy->cur);
 437                 }
 438         }
 439
 440         return 0;
 441 }
 442
 443 static void cpufreq_interactive_boost(void)
 444 {
 445         int i;
 446         int anyboost = 0;
 447         unsigned long flags;
 448         struct cpufreq_interactive_cpuinfo *pcpu;
 449
 450         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 451
 452         for_each_online_cpu(i) {
 453                 pcpu = &per_cpu(cpuinfo, i);
 454
 455                 if (pcpu->target_freq < hispeed_freq) {
 456                         pcpu->target_freq = hispeed_freq;
 457                         cpumask_set_cpu(i, &speedchange_cpumask);
 458                         pcpu->target_set_time_in_idle =
 459                                 get_cpu_idle_time_us(i, &pcpu->target_set_time);
 460                         pcpu->hispeed_validate_time = pcpu->target_set_time;
 461                         anyboost = 1;
 462                 }
 463
 464                 /*
 465                  * Set floor freq and (re)start timer for when last
 466                  * validated.
 467                  */
 468
 469                 pcpu->floor_freq = hispeed_freq;
 470                 pcpu->floor_validate_time = ktime_to_us(ktime_get());
 471         }
 472
 473         spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 474
 475         if (anyboost)
 476                 wake_up_process(speedchange_task);
 477 }
 478
 479 /*
 480  * Pulsed boost on input event raises CPUs to hispeed_freq and lets
 481  * usual algorithm of min_sample_time  decide when to allow speed
 482  * to drop.
 483  */
 484
 485 static void cpufreq_interactive_input_event(struct input_handle *handle,
 486                                             unsigned int type,
 487                                             unsigned int code, int value)
 488 {
 489         if (input_boost_val && type == EV_SYN && code == SYN_REPORT) {
 490                 trace_cpufreq_interactive_boost("input");
 491                 cpufreq_interactive_boost();
 492         }
 493 }
 494
 495 static void cpufreq_interactive_input_open(struct work_struct *w)
 496 {
 497         struct cpufreq_interactive_inputopen *io =
 498                 container_of(w, struct cpufreq_interactive_inputopen,
 499                              inputopen_work);
 500         int error;
 501
 502         error = input_open_device(io->handle);
 503         if (error)
 504                 input_unregister_handle(io->handle);
 505 }
 506
 507 static int cpufreq_interactive_input_connect(struct input_handler *handler,
 508                                              struct input_dev *dev,
 509                                              const struct input_device_id *id)
 510 {
 511         struct input_handle *handle;
 512         int error;
 513
 514         pr_info("%s: connect to %s\n", __func__, dev->name);
 515         handle = kzalloc(sizeof(struct input_handle), GFP_KERNEL);
 516         if (!handle)
 517                 return -ENOMEM;
 518
 519         handle->dev = dev;
 520         handle->handler = handler;
 521         handle->name = "cpufreq_interactive";
 522
 523         error = input_register_handle(handle);
 524         if (error)
 525                 goto err;
 526
 527         inputopen.handle = handle;
 528         queue_work(inputopen_wq, &inputopen.inputopen_work);
 529         return 0;
 530 err:
 531         kfree(handle);
 532         return error;
 533 }
 534
 535 static void cpufreq_interactive_input_disconnect(struct input_handle *handle)
 536 {
 537         input_close_device(handle);
 538         input_unregister_handle(handle);
 539         kfree(handle);
 540 }
 541
 542 static const struct input_device_id cpufreq_interactive_ids[] = {
 543         {
 544                 .flags = INPUT_DEVICE_ID_MATCH_EVBIT |
 545                          INPUT_DEVICE_ID_MATCH_ABSBIT,
 546                 .evbit = { BIT_MASK(EV_ABS) },
 547                 .absbit = { [BIT_WORD(ABS_MT_POSITION_X)] =
 548                             BIT_MASK(ABS_MT_POSITION_X) |
 549                             BIT_MASK(ABS_MT_POSITION_Y) },
 550         }, /* multi-touch touchscreen */
 551         {
 552                 .flags = INPUT_DEVICE_ID_MATCH_KEYBIT |
 553                          INPUT_DEVICE_ID_MATCH_ABSBIT,
 554                 .keybit = { [BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH) },
 555                 .absbit = { [BIT_WORD(ABS_X)] =
 556                             BIT_MASK(ABS_X) | BIT_MASK(ABS_Y) },
 557         }, /* touchpad */
 558         { },
 559 };
 560
 561 static struct input_handler cpufreq_interactive_input_handler = {
 562         .event          = cpufreq_interactive_input_event,
 563         .connect        = cpufreq_interactive_input_connect,
 564         .disconnect     = cpufreq_interactive_input_disconnect,
 565         .name           = "cpufreq_interactive",
 566         .id_table       = cpufreq_interactive_ids,
 567 };
 568
 569 static ssize_t show_hispeed_freq(struct kobject *kobj,
 570                                  struct attribute *attr, char *buf)
 571 {
 572         return sprintf(buf, "%llu\n", hispeed_freq);
 573 }
 574
 575 static ssize_t store_hispeed_freq(struct kobject *kobj,
 576                                   struct attribute *attr, const char *buf,
 577                                   size_t count)
 578 {
 579         int ret;
 580         u64 val;
 581
 582         ret = strict_strtoull(buf, 0, &val);
 583         if (ret < 0)
 584                 return ret;
 585         hispeed_freq = val;
 586         return count;
 587 }
 588
 589 static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644,
 590                 show_hispeed_freq, store_hispeed_freq);
 591
 592
 593 static ssize_t show_go_hispeed_load(struct kobject *kobj,
 594                                      struct attribute *attr, char *buf)
 595 {
 596         return sprintf(buf, "%lu\n", go_hispeed_load);
 597 }
 598
 599 static ssize_t store_go_hispeed_load(struct kobject *kobj,
 600                         struct attribute *attr, const char *buf, size_t count)
 601 {
 602         int ret;
 603         unsigned long val;
 604
 605         ret = strict_strtoul(buf, 0, &val);
 606         if (ret < 0)
 607                 return ret;
 608         go_hispeed_load = val;
 609         return count;
 610 }
 611
 612 static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644,
 613                 show_go_hispeed_load, store_go_hispeed_load);
 614
 615 static ssize_t show_min_sample_time(struct kobject *kobj,
 616                                 struct attribute *attr, char *buf)
 617 {
 618         return sprintf(buf, "%lu\n", min_sample_time);
 619 }
 620
 621 static ssize_t store_min_sample_time(struct kobject *kobj,
 622                         struct attribute *attr, const char *buf, size_t count)
 623 {
 624         int ret;
 625         unsigned long val;
 626
 627         ret = strict_strtoul(buf, 0, &val);
 628         if (ret < 0)
 629                 return ret;
 630         min_sample_time = val;
 631         return count;
 632 }
 633
 634 static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
 635                 show_min_sample_time, store_min_sample_time);
 636
 637 static ssize_t show_above_hispeed_delay(struct kobject *kobj,
 638                                         struct attribute *attr, char *buf)
 639 {
 640         return sprintf(buf, "%lu\n", above_hispeed_delay_val);
 641 }
 642
 643 static ssize_t store_above_hispeed_delay(struct kobject *kobj,
 644                                          struct attribute *attr,
 645                                          const char *buf, size_t count)
 646 {
 647         int ret;
 648         unsigned long val;
 649
 650         ret = strict_strtoul(buf, 0, &val);
 651         if (ret < 0)
 652                 return ret;
 653         above_hispeed_delay_val = val;
 654         return count;
 655 }
 656
 657 define_one_global_rw(above_hispeed_delay);
 658
 659 static ssize_t show_timer_rate(struct kobject *kobj,
 660                         struct attribute *attr, char *buf)
 661 {
 662         return sprintf(buf, "%lu\n", timer_rate);
 663 }
 664
 665 static ssize_t store_timer_rate(struct kobject *kobj,
 666                         struct attribute *attr, const char *buf, size_t count)
 667 {
 668         int ret;
 669         unsigned long val;
 670
 671         ret = strict_strtoul(buf, 0, &val);
 672         if (ret < 0)
 673                 return ret;
 674         timer_rate = val;
 675         return count;
 676 }
 677
 678 static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
 679                 show_timer_rate, store_timer_rate);
 680
 681 static ssize_t show_input_boost(struct kobject *kobj, struct attribute *attr,
 682                                 char *buf)
 683 {
 684         return sprintf(buf, "%u\n", input_boost_val);
 685 }
 686
 687 static ssize_t store_input_boost(struct kobject *kobj, struct attribute *attr,
 688                                  const char *buf, size_t count)
 689 {
 690         int ret;
 691         unsigned long val;
 692
 693         ret = strict_strtoul(buf, 0, &val);
 694         if (ret < 0)
 695                 return ret;
 696         input_boost_val = val;
 697         return count;
 698 }
 699
 700 define_one_global_rw(input_boost);
 701
 702 static ssize_t show_boost(struct kobject *kobj, struct attribute *attr,
 703                           char *buf)
 704 {
 705         return sprintf(buf, "%d\n", boost_val);
 706 }
 707
 708 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 709                            const char *buf, size_t count)
 710 {
 711         int ret;
 712         unsigned long val;
 713
 714         ret = kstrtoul(buf, 0, &val);
 715         if (ret < 0)
 716                 return ret;
 717
 718         boost_val = val;
 719
 720         if (boost_val) {
 721                 trace_cpufreq_interactive_boost("on");
 722                 cpufreq_interactive_boost();
 723         } else {
 724                 trace_cpufreq_interactive_unboost("off");
 725         }
 726
 727         return count;
 728 }
 729
 730 define_one_global_rw(boost);
 731
 732 static ssize_t store_boostpulse(struct kobject *kobj, struct attribute *attr,
 733                                 const char *buf, size_t count)
 734 {
 735         int ret;
 736         unsigned long val;
 737
 738         ret = kstrtoul(buf, 0, &val);
 739         if (ret < 0)
 740                 return ret;
 741
 742         trace_cpufreq_interactive_boost("pulse");
 743         cpufreq_interactive_boost();
 744         return count;
 745 }
 746
 747 static struct global_attr boostpulse =
 748         __ATTR(boostpulse, 0200, NULL, store_boostpulse);
 749
 750 static struct attribute *interactive_attributes[] = {
 751         &hispeed_freq_attr.attr,
 752         &go_hispeed_load_attr.attr,
 753         &above_hispeed_delay.attr,
 754         &min_sample_time_attr.attr,
 755         &timer_rate_attr.attr,
 756         &input_boost.attr,
 757         &boost.attr,
 758         &boostpulse.attr,
 759         NULL,
 760 };
 761
 762 static struct attribute_group interactive_attr_group = {
 763         .attrs = interactive_attributes,
 764         .name = "interactive",
 765 };
 766
 767 static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
 768                                              unsigned long val,
 769                                              void *data)
 770 {
 771         switch (val) {
 772         case IDLE_START:
 773                 cpufreq_interactive_idle_start();
 774                 break;
 775         case IDLE_END:
 776                 cpufreq_interactive_idle_end();
 777                 break;
 778         }
 779
 780         return 0;
 781 }
 782
 783 static struct notifier_block cpufreq_interactive_idle_nb = {
 784         .notifier_call = cpufreq_interactive_idle_notifier,
 785 };
 786
 787 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 788                 unsigned int event)
 789 {
 790         int rc;
 791         unsigned int j;
 792         struct cpufreq_interactive_cpuinfo *pcpu;
 793         struct cpufreq_frequency_table *freq_table;
 794
 795         switch (event) {
 796         case CPUFREQ_GOV_START:
 797                 if (!cpu_online(policy->cpu))
 798                         return -EINVAL;
 799
 800                 freq_table =
 801                         cpufreq_frequency_get_table(policy->cpu);
 802
 803                 for_each_cpu(j, policy->cpus) {
 804                         pcpu = &per_cpu(cpuinfo, j);
 805                         pcpu->policy = policy;
 806                         pcpu->target_freq = policy->cur;
 807                         pcpu->freq_table = freq_table;
 808                         pcpu->target_set_time_in_idle =
 809                                 get_cpu_idle_time_us(j,
 810                                              &pcpu->target_set_time);
 811                         pcpu->floor_freq = pcpu->target_freq;
 812                         pcpu->floor_validate_time =
 813                                 pcpu->target_set_time;
 814                         pcpu->hispeed_validate_time =
 815                                 pcpu->target_set_time;
 816                         pcpu->governor_enabled = 1;
 817                         smp_wmb();
 818                 }
 819
 820                 if (!hispeed_freq)
 821                         hispeed_freq = policy->max;
 822
 823                 /*
 824                  * Do not register the idle hook and create sysfs
 825                  * entries if we have already done so.
 826                  */
 827                 if (atomic_inc_return(&active_count) > 1)
 828                         return 0;
 829
 830                 rc = sysfs_create_group(cpufreq_global_kobject,
 831                                 &interactive_attr_group);
 832                 if (rc)
 833                         return rc;
 834
 835                 rc = input_register_handler(&cpufreq_interactive_input_handler);
 836                 if (rc)
 837                         pr_warn("%s: failed to register input handler\n",
 838                                 __func__);
 839
 840                 idle_notifier_register(&cpufreq_interactive_idle_nb);
 841                 break;
 842
 843         case CPUFREQ_GOV_STOP:
 844                 for_each_cpu(j, policy->cpus) {
 845                         pcpu = &per_cpu(cpuinfo, j);
 846                         pcpu->governor_enabled = 0;
 847                         smp_wmb();
 848                         del_timer_sync(&pcpu->cpu_timer);
 849
 850                         /*
 851                          * Reset idle exit time since we may cancel the timer
 852                          * before it can run after the last idle exit time,
 853                          * to avoid tripping the check in idle exit for a timer
 854                          * that is trying to run.
 855                          */
 856                         pcpu->idle_exit_time = 0;
 857                 }
 858
 859                 flush_work(&inputopen.inputopen_work);
 860                 if (atomic_dec_return(&active_count) > 0)
 861                         return 0;
 862
 863                 idle_notifier_unregister(&cpufreq_interactive_idle_nb);
 864                 input_unregister_handler(&cpufreq_interactive_input_handler);
 865                 sysfs_remove_group(cpufreq_global_kobject,
 866                                 &interactive_attr_group);
 867
 868                 break;
 869
 870         case CPUFREQ_GOV_LIMITS:
 871                 if (policy->max < policy->cur)
 872                         __cpufreq_driver_target(policy,
 873                                         policy->max, CPUFREQ_RELATION_H);
 874                 else if (policy->min > policy->cur)
 875                         __cpufreq_driver_target(policy,
 876                                         policy->min, CPUFREQ_RELATION_L);
 877                 break;
 878         }
 879         return 0;
 880 }
 881
 882 static int __init cpufreq_interactive_init(void)
 883 {
 884         unsigned int i;
 885         struct cpufreq_interactive_cpuinfo *pcpu;
 886         struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 887
 888         go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
 889         min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
 890         above_hispeed_delay_val = DEFAULT_ABOVE_HISPEED_DELAY;
 891         timer_rate = DEFAULT_TIMER_RATE;
 892
 893         /* Initalize per-cpu timers */
 894         for_each_possible_cpu(i) {
 895                 pcpu = &per_cpu(cpuinfo, i);
 896                 init_timer(&pcpu->cpu_timer);
 897                 pcpu->cpu_timer.function = cpufreq_interactive_timer;
 898                 pcpu->cpu_timer.data = i;
 899         }
 900
 901         spin_lock_init(&speedchange_cpumask_lock);
 902         speedchange_task =
 903                 kthread_create(cpufreq_interactive_speedchange_task, NULL,
 904                                "cfinteractive");
 905         if (IS_ERR(speedchange_task))
 906                 return PTR_ERR(speedchange_task);
 907
 908         sched_setscheduler_nocheck(speedchange_task, SCHED_FIFO, &param);
 909         get_task_struct(speedchange_task);
 910
 911         inputopen_wq = create_workqueue("cfinteractive");
 912
 913         if (!inputopen_wq)
 914                 goto err_freetask;
 915
 916         INIT_WORK(&inputopen.inputopen_work, cpufreq_interactive_input_open);
 917
 918         /* NB: wake up so the thread does not look hung to the freezer */
 919         wake_up_process(speedchange_task);
 920
 921         return cpufreq_register_governor(&cpufreq_gov_interactive);
 922
 923 err_freetask:
 924         put_task_struct(speedchange_task);
 925         return -ENOMEM;
 926 }
 927
 928 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 929 fs_initcall(cpufreq_interactive_init);
 930 #else
 931 module_init(cpufreq_interactive_init);
 932 #endif
 933
 934 static void __exit cpufreq_interactive_exit(void)
 935 {
 936         cpufreq_unregister_governor(&cpufreq_gov_interactive);
 937         kthread_stop(speedchange_task);
 938         put_task_struct(speedchange_task);
 939         destroy_workqueue(inputopen_wq);
 940 }
 941
 942 module_exit(cpufreq_interactive_exit);
 943
 944 MODULE_AUTHOR("Mike Chan <mike@android.com>");
 945 MODULE_DESCRIPTION("'cpufreq_interactive' - A cpufreq governor for "
 946         "Latency sensitive workloads");
 947 MODULE_LICENSE("GPL");