drivers/cpufreq/cpufreq_interactive.c

   1 /*
   2  * drivers/cpufreq/cpufreq_interactive.c
   3  *
   4  * Copyright (C) 2010 Google, Inc.
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * Author: Mike Chan (mike@android.com)
  16  *
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/cpumask.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/module.h>
  23 #include <linux/mutex.h>
  24 #include <linux/sched.h>
  25 #include <linux/sched/rt.h>
  26 #include <linux/tick.h>
  27 #include <linux/time.h>
  28 #include <linux/timer.h>
  29 #include <linux/workqueue.h>
  30 #include <linux/kthread.h>
  31 #include <linux/mutex.h>
  32 #include <linux/slab.h>
  33 #include <linux/input.h>
  34 #include <asm/cputime.h>
  35
  36 #define CREATE_TRACE_POINTS
  37 #include <trace/events/cpufreq_interactive.h>
  38
  39 static atomic_t active_count = ATOMIC_INIT(0);
  40
  41 struct cpufreq_interactive_cpuinfo {
  42         struct timer_list cpu_timer;
  43         int timer_idlecancel;
  44         u64 time_in_idle;
  45         u64 idle_exit_time;
  46         u64 timer_run_time;
  47         int idling;
  48         u64 target_set_time;
  49         u64 target_set_time_in_idle;
  50         struct cpufreq_policy *policy;
  51         struct cpufreq_frequency_table *freq_table;
  52         unsigned int target_freq;
  53         unsigned int floor_freq;
  54         u64 floor_validate_time;
  55         int governor_enabled;
  56 };
  57
  58 static DEFINE_PER_CPU(struct cpufreq_interactive_cpuinfo, cpuinfo);
  59
  60 /* Workqueues handle frequency scaling */
  61 static struct task_struct *up_task;
  62 static struct workqueue_struct *down_wq;
  63 static struct work_struct freq_scale_down_work;
  64 static cpumask_t up_cpumask;
  65 static spinlock_t up_cpumask_lock;
  66 static cpumask_t down_cpumask;
  67 static spinlock_t down_cpumask_lock;
  68 static struct mutex set_speed_lock;
  69
  70 /* Hi speed to bump to from lo speed when load burst (default max) */
  71 static u64 hispeed_freq;
  72
  73 /* Go to hi speed when CPU load at or above this value. */
  74 #define DEFAULT_GO_HISPEED_LOAD 85
  75 static unsigned long go_hispeed_load;
  76
  77 /*
  78  * The minimum amount of time to spend at a frequency before we can ramp down.
  79  */
  80 #define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
  81 static unsigned long min_sample_time;
  82
  83 /*
  84  * The sample rate of the timer used to increase frequency
  85  */
  86 #define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC)
  87 static unsigned long timer_rate;
  88
  89 /*
  90  * Wait this long before raising speed above hispeed, by default a single
  91  * timer interval.
  92  */
  93 #define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE
  94 static unsigned long above_hispeed_delay_val;
  95
  96 /*
  97  * Boost pulse to hispeed on touchscreen input.
  98  */
  99
 100 static int input_boost_val;
 101
 102 struct cpufreq_interactive_inputopen {
 103         struct input_handle *handle;
 104         struct work_struct inputopen_work;
 105 };
 106
 107 static struct cpufreq_interactive_inputopen inputopen;
 108
 109 /*
 110  * Non-zero means longer-term speed boost active.
 111  */
 112
 113 static int boost_val;
 114
 115 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 116                 unsigned int event);
 117
 118 #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 119 static
 120 #endif
 121 struct cpufreq_governor cpufreq_gov_interactive = {
 122         .name = "interactive",
 123         .governor = cpufreq_governor_interactive,
 124         .max_transition_latency = 10000000,
 125         .owner = THIS_MODULE,
 126 };
 127
 128 static void cpufreq_interactive_timer(unsigned long data)
 129 {
 130         unsigned int delta_idle;
 131         unsigned int delta_time;
 132         int cpu_load;
 133         int load_since_change;
 134         u64 time_in_idle;
 135         u64 idle_exit_time;
 136         struct cpufreq_interactive_cpuinfo *pcpu =
 137                 &per_cpu(cpuinfo, data);
 138         u64 now_idle;
 139         unsigned int new_freq;
 140         unsigned int index;
 141         unsigned long flags;
 142
 143         smp_rmb();
 144
 145         if (!pcpu->governor_enabled)
 146                 goto exit;
 147
 148         /*
 149          * Once pcpu->timer_run_time is updated to >= pcpu->idle_exit_time,
 150          * this lets idle exit know the current idle time sample has
 151          * been processed, and idle exit can generate a new sample and
 152          * re-arm the timer.  This prevents a concurrent idle
 153          * exit on that CPU from writing a new set of info at the same time
 154          * the timer function runs (the timer function can't use that info
 155          * until more time passes).
 156          */
 157         time_in_idle = pcpu->time_in_idle;
 158         idle_exit_time = pcpu->idle_exit_time;
 159         now_idle = get_cpu_idle_time_us(data, &pcpu->timer_run_time);
 160         smp_wmb();
 161
 162         /* If we raced with cancelling a timer, skip. */
 163         if (!idle_exit_time)
 164                 goto exit;
 165
 166         delta_idle = (unsigned int)(now_idle - time_in_idle);
 167         delta_time = (unsigned int)(pcpu->timer_run_time - idle_exit_time);
 168
 169         /*
 170          * If timer ran less than 1ms after short-term sample started, retry.
 171          */
 172         if (delta_time < 1000)
 173                 goto rearm;
 174
 175         if (delta_idle > delta_time)
 176                 cpu_load = 0;
 177         else
 178                 cpu_load = 100 * (delta_time - delta_idle) / delta_time;
 179
 180         delta_idle = (unsigned int)(now_idle - pcpu->target_set_time_in_idle);
 181         delta_time = (unsigned int)(pcpu->timer_run_time -
 182                                     pcpu->target_set_time);
 183
 184         if ((delta_time == 0) || (delta_idle > delta_time))
 185                 load_since_change = 0;
 186         else
 187                 load_since_change =
 188                         100 * (delta_time - delta_idle) / delta_time;
 189
 190         /*
 191          * Choose greater of short-term load (since last idle timer
 192          * started or timer function re-armed itself) or long-term load
 193          * (since last frequency change).
 194          */
 195         if (load_since_change > cpu_load)
 196                 cpu_load = load_since_change;
 197
 198         if (cpu_load >= go_hispeed_load || boost_val) {
 199                 if (pcpu->target_freq <= pcpu->policy->min) {
 200                         new_freq = hispeed_freq;
 201                 } else {
 202                         new_freq = pcpu->policy->max * cpu_load / 100;
 203
 204                         if (new_freq < hispeed_freq)
 205                                 new_freq = hispeed_freq;
 206
 207                         if (pcpu->target_freq == hispeed_freq &&
 208                             new_freq > hispeed_freq &&
 209                             pcpu->timer_run_time - pcpu->target_set_time
 210                             < above_hispeed_delay_val) {
 211                                 trace_cpufreq_interactive_notyet(data, cpu_load,
 212                                                                  pcpu->target_freq,
 213                                                                  new_freq);
 214                                 goto rearm;
 215                         }
 216                 }
 217         } else {
 218                 new_freq = pcpu->policy->max * cpu_load / 100;
 219         }
 220
 221         if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
 222                                            new_freq, CPUFREQ_RELATION_H,
 223                                            &index)) {
 224                 pr_warn_once("timer %d: cpufreq_frequency_table_target error\n",
 225                              (int) data);
 226                 goto rearm;
 227         }
 228
 229         new_freq = pcpu->freq_table[index].frequency;
 230
 231         /*
 232          * Do not scale below floor_freq unless we have been at or above the
 233          * floor frequency for the minimum sample time since last validated.
 234          */
 235         if (new_freq < pcpu->floor_freq) {
 236                 if (pcpu->timer_run_time - pcpu->target_validate_time
 237                     < min_sample_time) {
 238                         trace_cpufreq_interactive_notyet(data, cpu_load,
 239                                          pcpu->target_freq, new_freq);
 240                         goto rearm;
 241                 }
 242         }
 243
 244         pcpu->floor_freq = new_freq;
 245         pcpu->floor_validate_time = pcpu->timer_run_time;
 246
 247         if (pcpu->target_freq == new_freq) {
 248                 trace_cpufreq_interactive_already(data, cpu_load,
 249                                                   pcpu->target_freq, new_freq);
 250                 goto rearm_if_notmax;
 251         }
 252
 253         trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
 254                                          new_freq);
 255         pcpu->target_set_time_in_idle = now_idle;
 256         pcpu->target_set_time = pcpu->timer_run_time;
 257
 258         if (new_freq < pcpu->target_freq) {
 259                 pcpu->target_freq = new_freq;
 260                 spin_lock_irqsave(&down_cpumask_lock, flags);
 261                 cpumask_set_cpu(data, &down_cpumask);
 262                 spin_unlock_irqrestore(&down_cpumask_lock, flags);
 263                 queue_work(down_wq, &freq_scale_down_work);
 264         } else {
 265                 pcpu->target_freq = new_freq;
 266                 spin_lock_irqsave(&up_cpumask_lock, flags);
 267                 cpumask_set_cpu(data, &up_cpumask);
 268                 spin_unlock_irqrestore(&up_cpumask_lock, flags);
 269                 wake_up_process(up_task);
 270         }
 271
 272 rearm_if_notmax:
 273         /*
 274          * Already set max speed and don't see a need to change that,
 275          * wait until next idle to re-evaluate, don't need timer.
 276          */
 277         if (pcpu->target_freq == pcpu->policy->max)
 278                 goto exit;
 279
 280 rearm:
 281         if (!timer_pending(&pcpu->cpu_timer)) {
 282                 /*
 283                  * If already at min: if that CPU is idle, don't set timer.
 284                  * Else cancel the timer if that CPU goes idle.  We don't
 285                  * need to re-evaluate speed until the next idle exit.
 286                  */
 287                 if (pcpu->target_freq == pcpu->policy->min) {
 288                         smp_rmb();
 289
 290                         if (pcpu->idling)
 291                                 goto exit;
 292
 293                         pcpu->timer_idlecancel = 1;
 294                 }
 295
 296                 pcpu->time_in_idle = get_cpu_idle_time_us(
 297                         data, &pcpu->idle_exit_time);
 298                 mod_timer(&pcpu->cpu_timer,
 299                           jiffies + usecs_to_jiffies(timer_rate));
 300         }
 301
 302 exit:
 303         return;
 304 }
 305
 306 static void cpufreq_interactive_idle_start(void)
 307 {
 308         struct cpufreq_interactive_cpuinfo *pcpu =
 309                 &per_cpu(cpuinfo, smp_processor_id());
 310         int pending;
 311
 312         if (!pcpu->governor_enabled)
 313                 return;
 314
 315         pcpu->idling = 1;
 316         smp_wmb();
 317         pending = timer_pending(&pcpu->cpu_timer);
 318
 319         if (pcpu->target_freq != pcpu->policy->min) {
 320 #ifdef CONFIG_SMP
 321                 /*
 322                  * Entering idle while not at lowest speed.  On some
 323                  * platforms this can hold the other CPU(s) at that speed
 324                  * even though the CPU is idle. Set a timer to re-evaluate
 325                  * speed so this idle CPU doesn't hold the other CPUs above
 326                  * min indefinitely.  This should probably be a quirk of
 327                  * the CPUFreq driver.
 328                  */
 329                 if (!pending) {
 330                         pcpu->time_in_idle = get_cpu_idle_time_us(
 331                                 smp_processor_id(), &pcpu->idle_exit_time);
 332                         pcpu->timer_idlecancel = 0;
 333                         mod_timer(&pcpu->cpu_timer,
 334                                   jiffies + usecs_to_jiffies(timer_rate));
 335                 }
 336 #endif
 337         } else {
 338                 /*
 339                  * If at min speed and entering idle after load has
 340                  * already been evaluated, and a timer has been set just in
 341                  * case the CPU suddenly goes busy, cancel that timer.  The
 342                  * CPU didn't go busy; we'll recheck things upon idle exit.
 343                  */
 344                 if (pending && pcpu->timer_idlecancel) {
 345                         del_timer(&pcpu->cpu_timer);
 346                         /*
 347                          * Ensure last timer run time is after current idle
 348                          * sample start time, so next idle exit will always
 349                          * start a new idle sampling period.
 350                          */
 351                         pcpu->idle_exit_time = 0;
 352                         pcpu->timer_idlecancel = 0;
 353                 }
 354         }
 355
 356 }
 357
 358 static void cpufreq_interactive_idle_end(void)
 359 {
 360         struct cpufreq_interactive_cpuinfo *pcpu =
 361                 &per_cpu(cpuinfo, smp_processor_id());
 362
 363         pcpu->idling = 0;
 364         smp_wmb();
 365
 366         /*
 367          * Arm the timer for 1-2 ticks later if not already, and if the timer
 368          * function has already processed the previous load sampling
 369          * interval.  (If the timer is not pending but has not processed
 370          * the previous interval, it is probably racing with us on another
 371          * CPU.  Let it compute load based on the previous sample and then
 372          * re-arm the timer for another interval when it's done, rather
 373          * than updating the interval start time to be "now", which doesn't
 374          * give the timer function enough time to make a decision on this
 375          * run.)
 376          */
 377         if (timer_pending(&pcpu->cpu_timer) == 0 &&
 378             pcpu->timer_run_time >= pcpu->idle_exit_time &&
 379             pcpu->governor_enabled) {
 380                 pcpu->time_in_idle =
 381                         get_cpu_idle_time_us(smp_processor_id(),
 382                                              &pcpu->idle_exit_time);
 383                 pcpu->timer_idlecancel = 0;
 384                 mod_timer(&pcpu->cpu_timer,
 385                           jiffies + usecs_to_jiffies(timer_rate));
 386         }
 387
 388 }
 389
 390 static int cpufreq_interactive_up_task(void *data)
 391 {
 392         unsigned int cpu;
 393         cpumask_t tmp_mask;
 394         unsigned long flags;
 395         struct cpufreq_interactive_cpuinfo *pcpu;
 396
 397         while (1) {
 398                 set_current_state(TASK_INTERRUPTIBLE);
 399                 spin_lock_irqsave(&up_cpumask_lock, flags);
 400
 401                 if (cpumask_empty(&up_cpumask)) {
 402                         spin_unlock_irqrestore(&up_cpumask_lock, flags);
 403                         schedule();
 404
 405                         if (kthread_should_stop())
 406                                 break;
 407
 408                         spin_lock_irqsave(&up_cpumask_lock, flags);
 409                 }
 410
 411                 set_current_state(TASK_RUNNING);
 412                 tmp_mask = up_cpumask;
 413                 cpumask_clear(&up_cpumask);
 414                 spin_unlock_irqrestore(&up_cpumask_lock, flags);
 415
 416                 for_each_cpu(cpu, &tmp_mask) {
 417                         unsigned int j;
 418                         unsigned int max_freq = 0;
 419
 420                         pcpu = &per_cpu(cpuinfo, cpu);
 421                         smp_rmb();
 422
 423                         if (!pcpu->governor_enabled)
 424                                 continue;
 425
 426                         mutex_lock(&set_speed_lock);
 427
 428                         for_each_cpu(j, pcpu->policy->cpus) {
 429                                 struct cpufreq_interactive_cpuinfo *pjcpu =
 430                                         &per_cpu(cpuinfo, j);
 431
 432                                 if (pjcpu->target_freq > max_freq)
 433                                         max_freq = pjcpu->target_freq;
 434                         }
 435
 436                         if (max_freq != pcpu->policy->cur)
 437                                 __cpufreq_driver_target(pcpu->policy,
 438                                                         max_freq,
 439                                                         CPUFREQ_RELATION_H);
 440                         mutex_unlock(&set_speed_lock);
 441                         trace_cpufreq_interactive_up(cpu, pcpu->target_freq,
 442                                                      pcpu->policy->cur);
 443                 }
 444         }
 445
 446         return 0;
 447 }
 448
 449 static void cpufreq_interactive_freq_down(struct work_struct *work)
 450 {
 451         unsigned int cpu;
 452         cpumask_t tmp_mask;
 453         unsigned long flags;
 454         struct cpufreq_interactive_cpuinfo *pcpu;
 455
 456         spin_lock_irqsave(&down_cpumask_lock, flags);
 457         tmp_mask = down_cpumask;
 458         cpumask_clear(&down_cpumask);
 459         spin_unlock_irqrestore(&down_cpumask_lock, flags);
 460
 461         for_each_cpu(cpu, &tmp_mask) {
 462                 unsigned int j;
 463                 unsigned int max_freq = 0;
 464
 465                 pcpu = &per_cpu(cpuinfo, cpu);
 466                 smp_rmb();
 467
 468                 if (!pcpu->governor_enabled)
 469                         continue;
 470
 471                 mutex_lock(&set_speed_lock);
 472
 473                 for_each_cpu(j, pcpu->policy->cpus) {
 474                         struct cpufreq_interactive_cpuinfo *pjcpu =
 475                                 &per_cpu(cpuinfo, j);
 476
 477                         if (pjcpu->target_freq > max_freq)
 478                                 max_freq = pjcpu->target_freq;
 479                 }
 480
 481                 if (max_freq != pcpu->policy->cur)
 482                         __cpufreq_driver_target(pcpu->policy, max_freq,
 483                                                 CPUFREQ_RELATION_H);
 484
 485                 mutex_unlock(&set_speed_lock);
 486                 trace_cpufreq_interactive_down(cpu, pcpu->target_freq,
 487                                                pcpu->policy->cur);
 488         }
 489 }
 490
 491 static void cpufreq_interactive_boost(void)
 492 {
 493         int i;
 494         int anyboost = 0;
 495         unsigned long flags;
 496         struct cpufreq_interactive_cpuinfo *pcpu;
 497
 498         spin_lock_irqsave(&up_cpumask_lock, flags);
 499
 500         for_each_online_cpu(i) {
 501                 pcpu = &per_cpu(cpuinfo, i);
 502
 503                 if (pcpu->target_freq < hispeed_freq) {
 504                         pcpu->target_freq = hispeed_freq;
 505                         cpumask_set_cpu(i, &up_cpumask);
 506                         pcpu->target_set_time_in_idle =
 507                                 get_cpu_idle_time_us(i, &pcpu->target_set_time);
 508                         anyboost = 1;
 509                 }
 510
 511                 /*
 512                  * Set floor freq and (re)start timer for when last
 513                  * validated.
 514                  */
 515
 516                 pcpu->floor_freq = hispeed_freq;
 517                 pcpu->floor_validate_time = ktime_to_us(ktime_get());
 518         }
 519
 520         spin_unlock_irqrestore(&up_cpumask_lock, flags);
 521
 522         if (anyboost)
 523                 wake_up_process(up_task);
 524 }
 525
 526 /*
 527  * Pulsed boost on input event raises CPUs to hispeed_freq and lets
 528  * usual algorithm of min_sample_time  decide when to allow speed
 529  * to drop.
 530  */
 531
 532 static void cpufreq_interactive_input_event(struct input_handle *handle,
 533                                             unsigned int type,
 534                                             unsigned int code, int value)
 535 {
 536         if (input_boost_val && type == EV_SYN && code == SYN_REPORT) {
 537                 trace_cpufreq_interactive_boost("input");
 538                 cpufreq_interactive_boost();
 539         }
 540 }
 541
 542 static void cpufreq_interactive_input_open(struct work_struct *w)
 543 {
 544         struct cpufreq_interactive_inputopen *io =
 545                 container_of(w, struct cpufreq_interactive_inputopen,
 546                              inputopen_work);
 547         int error;
 548
 549         error = input_open_device(io->handle);
 550         if (error)
 551                 input_unregister_handle(io->handle);
 552 }
 553
 554 static int cpufreq_interactive_input_connect(struct input_handler *handler,
 555                                              struct input_dev *dev,
 556                                              const struct input_device_id *id)
 557 {
 558         struct input_handle *handle;
 559         int error;
 560
 561         pr_info("%s: connect to %s\n", __func__, dev->name);
 562         handle = kzalloc(sizeof(struct input_handle), GFP_KERNEL);
 563         if (!handle)
 564                 return -ENOMEM;
 565
 566         handle->dev = dev;
 567         handle->handler = handler;
 568         handle->name = "cpufreq_interactive";
 569
 570         error = input_register_handle(handle);
 571         if (error)
 572                 goto err;
 573
 574         inputopen.handle = handle;
 575         queue_work(down_wq, &inputopen.inputopen_work);
 576         return 0;
 577 err:
 578         kfree(handle);
 579         return error;
 580 }
 581
 582 static void cpufreq_interactive_input_disconnect(struct input_handle *handle)
 583 {
 584         input_close_device(handle);
 585         input_unregister_handle(handle);
 586         kfree(handle);
 587 }
 588
 589 static const struct input_device_id cpufreq_interactive_ids[] = {
 590         {
 591                 .flags = INPUT_DEVICE_ID_MATCH_EVBIT |
 592                          INPUT_DEVICE_ID_MATCH_ABSBIT,
 593                 .evbit = { BIT_MASK(EV_ABS) },
 594                 .absbit = { [BIT_WORD(ABS_MT_POSITION_X)] =
 595                             BIT_MASK(ABS_MT_POSITION_X) |
 596                             BIT_MASK(ABS_MT_POSITION_Y) },
 597         }, /* multi-touch touchscreen */
 598         {
 599                 .flags = INPUT_DEVICE_ID_MATCH_KEYBIT |
 600                          INPUT_DEVICE_ID_MATCH_ABSBIT,
 601                 .keybit = { [BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH) },
 602                 .absbit = { [BIT_WORD(ABS_X)] =
 603                             BIT_MASK(ABS_X) | BIT_MASK(ABS_Y) },
 604         }, /* touchpad */
 605         { },
 606 };
 607
 608 static struct input_handler cpufreq_interactive_input_handler = {
 609         .event          = cpufreq_interactive_input_event,
 610         .connect        = cpufreq_interactive_input_connect,
 611         .disconnect     = cpufreq_interactive_input_disconnect,
 612         .name           = "cpufreq_interactive",
 613         .id_table       = cpufreq_interactive_ids,
 614 };
 615
 616 static ssize_t show_hispeed_freq(struct kobject *kobj,
 617                                  struct attribute *attr, char *buf)
 618 {
 619         return sprintf(buf, "%llu\n", hispeed_freq);
 620 }
 621
 622 static ssize_t store_hispeed_freq(struct kobject *kobj,
 623                                   struct attribute *attr, const char *buf,
 624                                   size_t count)
 625 {
 626         int ret;
 627         u64 val;
 628
 629         ret = strict_strtoull(buf, 0, &val);
 630         if (ret < 0)
 631                 return ret;
 632         hispeed_freq = val;
 633         return count;
 634 }
 635
 636 static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644,
 637                 show_hispeed_freq, store_hispeed_freq);
 638
 639
 640 static ssize_t show_go_hispeed_load(struct kobject *kobj,
 641                                      struct attribute *attr, char *buf)
 642 {
 643         return sprintf(buf, "%lu\n", go_hispeed_load);
 644 }
 645
 646 static ssize_t store_go_hispeed_load(struct kobject *kobj,
 647                         struct attribute *attr, const char *buf, size_t count)
 648 {
 649         int ret;
 650         unsigned long val;
 651
 652         ret = strict_strtoul(buf, 0, &val);
 653         if (ret < 0)
 654                 return ret;
 655         go_hispeed_load = val;
 656         return count;
 657 }
 658
 659 static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644,
 660                 show_go_hispeed_load, store_go_hispeed_load);
 661
 662 static ssize_t show_min_sample_time(struct kobject *kobj,
 663                                 struct attribute *attr, char *buf)
 664 {
 665         return sprintf(buf, "%lu\n", min_sample_time);
 666 }
 667
 668 static ssize_t store_min_sample_time(struct kobject *kobj,
 669                         struct attribute *attr, const char *buf, size_t count)
 670 {
 671         int ret;
 672         unsigned long val;
 673
 674         ret = strict_strtoul(buf, 0, &val);
 675         if (ret < 0)
 676                 return ret;
 677         min_sample_time = val;
 678         return count;
 679 }
 680
 681 static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
 682                 show_min_sample_time, store_min_sample_time);
 683
 684 static ssize_t show_above_hispeed_delay(struct kobject *kobj,
 685                                         struct attribute *attr, char *buf)
 686 {
 687         return sprintf(buf, "%lu\n", above_hispeed_delay_val);
 688 }
 689
 690 static ssize_t store_above_hispeed_delay(struct kobject *kobj,
 691                                          struct attribute *attr,
 692                                          const char *buf, size_t count)
 693 {
 694         int ret;
 695         unsigned long val;
 696
 697         ret = strict_strtoul(buf, 0, &val);
 698         if (ret < 0)
 699                 return ret;
 700         above_hispeed_delay_val = val;
 701         return count;
 702 }
 703
 704 define_one_global_rw(above_hispeed_delay);
 705
 706 static ssize_t show_timer_rate(struct kobject *kobj,
 707                         struct attribute *attr, char *buf)
 708 {
 709         return sprintf(buf, "%lu\n", timer_rate);
 710 }
 711
 712 static ssize_t store_timer_rate(struct kobject *kobj,
 713                         struct attribute *attr, const char *buf, size_t count)
 714 {
 715         int ret;
 716         unsigned long val;
 717
 718         ret = strict_strtoul(buf, 0, &val);
 719         if (ret < 0)
 720                 return ret;
 721         timer_rate = val;
 722         return count;
 723 }
 724
 725 static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
 726                 show_timer_rate, store_timer_rate);
 727
 728 static ssize_t show_input_boost(struct kobject *kobj, struct attribute *attr,
 729                                 char *buf)
 730 {
 731         return sprintf(buf, "%u\n", input_boost_val);
 732 }
 733
 734 static ssize_t store_input_boost(struct kobject *kobj, struct attribute *attr,
 735                                  const char *buf, size_t count)
 736 {
 737         int ret;
 738         unsigned long val;
 739
 740         ret = strict_strtoul(buf, 0, &val);
 741         if (ret < 0)
 742                 return ret;
 743         input_boost_val = val;
 744         return count;
 745 }
 746
 747 define_one_global_rw(input_boost);
 748
 749 static ssize_t show_boost(struct kobject *kobj, struct attribute *attr,
 750                           char *buf)
 751 {
 752         return sprintf(buf, "%d\n", boost_val);
 753 }
 754
 755 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 756                            const char *buf, size_t count)
 757 {
 758         int ret;
 759         unsigned long val;
 760
 761         ret = kstrtoul(buf, 0, &val);
 762         if (ret < 0)
 763                 return ret;
 764
 765         boost_val = val;
 766
 767         if (boost_val) {
 768                 trace_cpufreq_interactive_boost("on");
 769                 cpufreq_interactive_boost();
 770         } else {
 771                 trace_cpufreq_interactive_unboost("off");
 772         }
 773
 774         return count;
 775 }
 776
 777 define_one_global_rw(boost);
 778
 779 static ssize_t store_boostpulse(struct kobject *kobj, struct attribute *attr,
 780                                 const char *buf, size_t count)
 781 {
 782         int ret;
 783         unsigned long val;
 784
 785         ret = kstrtoul(buf, 0, &val);
 786         if (ret < 0)
 787                 return ret;
 788
 789         trace_cpufreq_interactive_boost("pulse");
 790         cpufreq_interactive_boost();
 791         return count;
 792 }
 793
 794 static struct global_attr boostpulse =
 795         __ATTR(boostpulse, 0200, NULL, store_boostpulse);
 796
 797 static struct attribute *interactive_attributes[] = {
 798         &hispeed_freq_attr.attr,
 799         &go_hispeed_load_attr.attr,
 800         &above_hispeed_delay.attr,
 801         &min_sample_time_attr.attr,
 802         &timer_rate_attr.attr,
 803         &input_boost.attr,
 804         &boost.attr,
 805         &boostpulse.attr,
 806         NULL,
 807 };
 808
 809 static struct attribute_group interactive_attr_group = {
 810         .attrs = interactive_attributes,
 811         .name = "interactive",
 812 };
 813
 814 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 815                 unsigned int event)
 816 {
 817         int rc;
 818         unsigned int j;
 819         struct cpufreq_interactive_cpuinfo *pcpu;
 820         struct cpufreq_frequency_table *freq_table;
 821
 822         switch (event) {
 823         case CPUFREQ_GOV_START:
 824                 if (!cpu_online(policy->cpu))
 825                         return -EINVAL;
 826
 827                 freq_table =
 828                         cpufreq_frequency_get_table(policy->cpu);
 829
 830                 for_each_cpu(j, policy->cpus) {
 831                         pcpu = &per_cpu(cpuinfo, j);
 832                         pcpu->policy = policy;
 833                         pcpu->target_freq = policy->cur;
 834                         pcpu->freq_table = freq_table;
 835                         pcpu->target_set_time_in_idle =
 836                                 get_cpu_idle_time_us(j,
 837                                              &pcpu->target_set_time);
 838                         pcpu->floor_freq = pcpu->target_freq;
 839                         pcpu->floor_validate_time =
 840                                 pcpu->target_set_time;
 841                         pcpu->governor_enabled = 1;
 842                         smp_wmb();
 843                 }
 844
 845                 if (!hispeed_freq)
 846                         hispeed_freq = policy->max;
 847
 848                 /*
 849                  * Do not register the idle hook and create sysfs
 850                  * entries if we have already done so.
 851                  */
 852                 if (atomic_inc_return(&active_count) > 1)
 853                         return 0;
 854
 855                 rc = sysfs_create_group(cpufreq_global_kobject,
 856                                 &interactive_attr_group);
 857                 if (rc)
 858                         return rc;
 859
 860                 rc = input_register_handler(&cpufreq_interactive_input_handler);
 861                 if (rc)
 862                         pr_warn("%s: failed to register input handler\n",
 863                                 __func__);
 864
 865                 break;
 866
 867         case CPUFREQ_GOV_STOP:
 868                 for_each_cpu(j, policy->cpus) {
 869                         pcpu = &per_cpu(cpuinfo, j);
 870                         pcpu->governor_enabled = 0;
 871                         smp_wmb();
 872                         del_timer_sync(&pcpu->cpu_timer);
 873
 874                         /*
 875                          * Reset idle exit time since we may cancel the timer
 876                          * before it can run after the last idle exit time,
 877                          * to avoid tripping the check in idle exit for a timer
 878                          * that is trying to run.
 879                          */
 880                         pcpu->idle_exit_time = 0;
 881                 }
 882
 883                 flush_work(&freq_scale_down_work);
 884                 if (atomic_dec_return(&active_count) > 0)
 885                         return 0;
 886
 887                 input_unregister_handler(&cpufreq_interactive_input_handler);
 888                 sysfs_remove_group(cpufreq_global_kobject,
 889                                 &interactive_attr_group);
 890
 891                 break;
 892
 893         case CPUFREQ_GOV_LIMITS:
 894                 if (policy->max < policy->cur)
 895                         __cpufreq_driver_target(policy,
 896                                         policy->max, CPUFREQ_RELATION_H);
 897                 else if (policy->min > policy->cur)
 898                         __cpufreq_driver_target(policy,
 899                                         policy->min, CPUFREQ_RELATION_L);
 900                 break;
 901         }
 902         return 0;
 903 }
 904
 905 static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
 906                                              unsigned long val,
 907                                              void *data)
 908 {
 909         switch (val) {
 910         case IDLE_START:
 911                 cpufreq_interactive_idle_start();
 912                 break;
 913         case IDLE_END:
 914                 cpufreq_interactive_idle_end();
 915                 break;
 916         }
 917
 918         return 0;
 919 }
 920
 921 static struct notifier_block cpufreq_interactive_idle_nb = {
 922         .notifier_call = cpufreq_interactive_idle_notifier,
 923 };
 924
 925 static int __init cpufreq_interactive_init(void)
 926 {
 927         unsigned int i;
 928         struct cpufreq_interactive_cpuinfo *pcpu;
 929         struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 930
 931         go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
 932         min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
 933         above_hispeed_delay_val = DEFAULT_ABOVE_HISPEED_DELAY;
 934         timer_rate = DEFAULT_TIMER_RATE;
 935
 936         /* Initalize per-cpu timers */
 937         for_each_possible_cpu(i) {
 938                 pcpu = &per_cpu(cpuinfo, i);
 939                 init_timer(&pcpu->cpu_timer);
 940                 pcpu->cpu_timer.function = cpufreq_interactive_timer;
 941                 pcpu->cpu_timer.data = i;
 942         }
 943
 944         up_task = kthread_create(cpufreq_interactive_up_task, NULL,
 945                                  "kinteractiveup");
 946         if (IS_ERR(up_task))
 947                 return PTR_ERR(up_task);
 948
 949         sched_setscheduler_nocheck(up_task, SCHED_FIFO, &param);
 950         get_task_struct(up_task);
 951
 952         /* No rescuer thread, bind to CPU queuing the work for possibly
 953            warm cache (probably doesn't matter much). */
 954         down_wq = alloc_workqueue("knteractive_down", 0, 1);
 955
 956         if (!down_wq)
 957                 goto err_freeuptask;
 958
 959         INIT_WORK(&freq_scale_down_work,
 960                   cpufreq_interactive_freq_down);
 961
 962         spin_lock_init(&up_cpumask_lock);
 963         spin_lock_init(&down_cpumask_lock);
 964         mutex_init(&set_speed_lock);
 965
 966         idle_notifier_register(&cpufreq_interactive_idle_nb);
 967         INIT_WORK(&inputopen.inputopen_work, cpufreq_interactive_input_open);
 968         return cpufreq_register_governor(&cpufreq_gov_interactive);
 969
 970 err_freeuptask:
 971         put_task_struct(up_task);
 972         return -ENOMEM;
 973 }
 974
 975 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 976 fs_initcall(cpufreq_interactive_init);
 977 #else
 978 module_init(cpufreq_interactive_init);
 979 #endif
 980
 981 static void __exit cpufreq_interactive_exit(void)
 982 {
 983         cpufreq_unregister_governor(&cpufreq_gov_interactive);
 984         kthread_stop(up_task);
 985         put_task_struct(up_task);
 986         destroy_workqueue(down_wq);
 987 }
 988
 989 module_exit(cpufreq_interactive_exit);
 990
 991 MODULE_AUTHOR("Mike Chan <mike@android.com>");
 992 MODULE_DESCRIPTION("'cpufreq_interactive' - A cpufreq governor for "
 993         "Latency sensitive workloads");
 994 MODULE_LICENSE("GPL");