drivers/cpufreq/cpufreq_interactive.c

   1 /*
   2  * drivers/cpufreq/cpufreq_interactive.c
   3  *
   4  * Copyright (C) 2010 Google, Inc.
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * Author: Mike Chan (mike@android.com)
  16  *
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/cpumask.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/module.h>
  23 #include <linux/mutex.h>
  24 #include <linux/sched.h>
  25 #include <linux/sched/rt.h>
  26 #include <linux/tick.h>
  27 #include <linux/time.h>
  28 #include <linux/timer.h>
  29 #include <linux/workqueue.h>
  30 #include <linux/kthread.h>
  31 #include <linux/mutex.h>
  32 #include <linux/slab.h>
  33 #include <linux/input.h>
  34 #include <asm/cputime.h>
  35
  36 #define CREATE_TRACE_POINTS
  37 #include <trace/events/cpufreq_interactive.h>
  38
  39 static atomic_t active_count = ATOMIC_INIT(0);
  40
  41 struct cpufreq_interactive_cpuinfo {
  42         struct timer_list cpu_timer;
  43         int timer_idlecancel;
  44         u64 time_in_idle;
  45         u64 idle_exit_time;
  46         u64 timer_run_time;
  47         int idling;
  48         u64 target_set_time;
  49         u64 target_set_time_in_idle;
  50         struct cpufreq_policy *policy;
  51         struct cpufreq_frequency_table *freq_table;
  52         unsigned int target_freq;
  53         unsigned int floor_freq;
  54         u64 floor_validate_time;
  55         u64 hispeed_validate_time;
  56         int governor_enabled;
  57 };
  58
  59 static DEFINE_PER_CPU(struct cpufreq_interactive_cpuinfo, cpuinfo);
  60
  61 /* Workqueues handle frequency scaling */
  62 static struct task_struct *up_task;
  63 static struct workqueue_struct *down_wq;
  64 static struct work_struct freq_scale_down_work;
  65 static cpumask_t up_cpumask;
  66 static spinlock_t up_cpumask_lock;
  67 static cpumask_t down_cpumask;
  68 static spinlock_t down_cpumask_lock;
  69 static struct mutex set_speed_lock;
  70
  71 /* Hi speed to bump to from lo speed when load burst (default max) */
  72 static u64 hispeed_freq;
  73
  74 /* Go to hi speed when CPU load at or above this value. */
  75 #define DEFAULT_GO_HISPEED_LOAD 85
  76 static unsigned long go_hispeed_load;
  77
  78 /*
  79  * The minimum amount of time to spend at a frequency before we can ramp down.
  80  */
  81 #define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
  82 static unsigned long min_sample_time;
  83
  84 /*
  85  * The sample rate of the timer used to increase frequency
  86  */
  87 #define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC)
  88 static unsigned long timer_rate;
  89
  90 /*
  91  * Wait this long before raising speed above hispeed, by default a single
  92  * timer interval.
  93  */
  94 #define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE
  95 static unsigned long above_hispeed_delay_val;
  96
  97 /*
  98  * Boost pulse to hispeed on touchscreen input.
  99  */
 100
 101 static int input_boost_val;
 102
 103 struct cpufreq_interactive_inputopen {
 104         struct input_handle *handle;
 105         struct work_struct inputopen_work;
 106 };
 107
 108 static struct cpufreq_interactive_inputopen inputopen;
 109
 110 /*
 111  * Non-zero means longer-term speed boost active.
 112  */
 113
 114 static int boost_val;
 115
 116 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 117                 unsigned int event);
 118
 119 #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 120 static
 121 #endif
 122 struct cpufreq_governor cpufreq_gov_interactive = {
 123         .name = "interactive",
 124         .governor = cpufreq_governor_interactive,
 125         .max_transition_latency = 10000000,
 126         .owner = THIS_MODULE,
 127 };
 128
 129 static void cpufreq_interactive_timer(unsigned long data)
 130 {
 131         unsigned int delta_idle;
 132         unsigned int delta_time;
 133         int cpu_load;
 134         int load_since_change;
 135         u64 time_in_idle;
 136         u64 idle_exit_time;
 137         struct cpufreq_interactive_cpuinfo *pcpu =
 138                 &per_cpu(cpuinfo, data);
 139         u64 now_idle;
 140         unsigned int new_freq;
 141         unsigned int index;
 142         unsigned long flags;
 143
 144         smp_rmb();
 145
 146         if (!pcpu->governor_enabled)
 147                 goto exit;
 148
 149         /*
 150          * Once pcpu->timer_run_time is updated to >= pcpu->idle_exit_time,
 151          * this lets idle exit know the current idle time sample has
 152          * been processed, and idle exit can generate a new sample and
 153          * re-arm the timer.  This prevents a concurrent idle
 154          * exit on that CPU from writing a new set of info at the same time
 155          * the timer function runs (the timer function can't use that info
 156          * until more time passes).
 157          */
 158         time_in_idle = pcpu->time_in_idle;
 159         idle_exit_time = pcpu->idle_exit_time;
 160         now_idle = get_cpu_idle_time_us(data, &pcpu->timer_run_time);
 161         smp_wmb();
 162
 163         /* If we raced with cancelling a timer, skip. */
 164         if (!idle_exit_time)
 165                 goto exit;
 166
 167         delta_idle = (unsigned int)(now_idle - time_in_idle);
 168         delta_time = (unsigned int)(pcpu->timer_run_time - idle_exit_time);
 169
 170         /*
 171          * If timer ran less than 1ms after short-term sample started, retry.
 172          */
 173         if (delta_time < 1000)
 174                 goto rearm;
 175
 176         if (delta_idle > delta_time)
 177                 cpu_load = 0;
 178         else
 179                 cpu_load = 100 * (delta_time - delta_idle) / delta_time;
 180
 181         delta_idle = (unsigned int)(now_idle - pcpu->target_set_time_in_idle);
 182         delta_time = (unsigned int)(pcpu->timer_run_time -
 183                                     pcpu->target_set_time);
 184
 185         if ((delta_time == 0) || (delta_idle > delta_time))
 186                 load_since_change = 0;
 187         else
 188                 load_since_change =
 189                         100 * (delta_time - delta_idle) / delta_time;
 190
 191         /*
 192          * Choose greater of short-term load (since last idle timer
 193          * started or timer function re-armed itself) or long-term load
 194          * (since last frequency change).
 195          */
 196         if (load_since_change > cpu_load)
 197                 cpu_load = load_since_change;
 198
 199         if (cpu_load >= go_hispeed_load || boost_val) {
 200                 if (pcpu->target_freq <= pcpu->policy->min) {
 201                         new_freq = hispeed_freq;
 202                 } else {
 203                         new_freq = pcpu->policy->max * cpu_load / 100;
 204
 205                         if (new_freq < hispeed_freq)
 206                                 new_freq = hispeed_freq;
 207
 208                         if (pcpu->target_freq == hispeed_freq &&
 209                             new_freq > hispeed_freq &&
 210                             pcpu->timer_run_time - pcpu->hispeed_validate_time
 211                             < above_hispeed_delay_val) {
 212                                 trace_cpufreq_interactive_notyet(data, cpu_load,
 213                                                                  pcpu->target_freq,
 214                                                                  new_freq);
 215                                 goto rearm;
 216                         }
 217                 }
 218         } else {
 219                 new_freq = pcpu->policy->max * cpu_load / 100;
 220         }
 221
 222         if (new_freq <= hispeed_freq)
 223                 pcpu->hispeed_validate_time = pcpu->timer_run_time;
 224
 225         if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
 226                                            new_freq, CPUFREQ_RELATION_H,
 227                                            &index)) {
 228                 pr_warn_once("timer %d: cpufreq_frequency_table_target error\n",
 229                              (int) data);
 230                 goto rearm;
 231         }
 232
 233         new_freq = pcpu->freq_table[index].frequency;
 234
 235         /*
 236          * Do not scale below floor_freq unless we have been at or above the
 237          * floor frequency for the minimum sample time since last validated.
 238          */
 239         if (new_freq < pcpu->floor_freq) {
 240                 if (pcpu->timer_run_time - pcpu->floor_validate_time
 241                     < min_sample_time) {
 242                         trace_cpufreq_interactive_notyet(data, cpu_load,
 243                                          pcpu->target_freq, new_freq);
 244                         goto rearm;
 245                 }
 246         }
 247
 248         pcpu->floor_freq = new_freq;
 249         pcpu->floor_validate_time = pcpu->timer_run_time;
 250
 251         if (pcpu->target_freq == new_freq) {
 252                 trace_cpufreq_interactive_already(data, cpu_load,
 253                                                   pcpu->target_freq, new_freq);
 254                 goto rearm_if_notmax;
 255         }
 256
 257         trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
 258                                          new_freq);
 259         pcpu->target_set_time_in_idle = now_idle;
 260         pcpu->target_set_time = pcpu->timer_run_time;
 261
 262         if (new_freq < pcpu->target_freq) {
 263                 pcpu->target_freq = new_freq;
 264                 spin_lock_irqsave(&down_cpumask_lock, flags);
 265                 cpumask_set_cpu(data, &down_cpumask);
 266                 spin_unlock_irqrestore(&down_cpumask_lock, flags);
 267                 queue_work(down_wq, &freq_scale_down_work);
 268         } else {
 269                 pcpu->target_freq = new_freq;
 270                 spin_lock_irqsave(&up_cpumask_lock, flags);
 271                 cpumask_set_cpu(data, &up_cpumask);
 272                 spin_unlock_irqrestore(&up_cpumask_lock, flags);
 273                 wake_up_process(up_task);
 274         }
 275
 276 rearm_if_notmax:
 277         /*
 278          * Already set max speed and don't see a need to change that,
 279          * wait until next idle to re-evaluate, don't need timer.
 280          */
 281         if (pcpu->target_freq == pcpu->policy->max)
 282                 goto exit;
 283
 284 rearm:
 285         if (!timer_pending(&pcpu->cpu_timer)) {
 286                 /*
 287                  * If already at min: if that CPU is idle, don't set timer.
 288                  * Else cancel the timer if that CPU goes idle.  We don't
 289                  * need to re-evaluate speed until the next idle exit.
 290                  */
 291                 if (pcpu->target_freq == pcpu->policy->min) {
 292                         smp_rmb();
 293
 294                         if (pcpu->idling)
 295                                 goto exit;
 296
 297                         pcpu->timer_idlecancel = 1;
 298                 }
 299
 300                 pcpu->time_in_idle = get_cpu_idle_time_us(
 301                         data, &pcpu->idle_exit_time);
 302                 mod_timer(&pcpu->cpu_timer,
 303                           jiffies + usecs_to_jiffies(timer_rate));
 304         }
 305
 306 exit:
 307         return;
 308 }
 309
 310 static void cpufreq_interactive_idle_start(void)
 311 {
 312         struct cpufreq_interactive_cpuinfo *pcpu =
 313                 &per_cpu(cpuinfo, smp_processor_id());
 314         int pending;
 315
 316         if (!pcpu->governor_enabled)
 317                 return;
 318
 319         pcpu->idling = 1;
 320         smp_wmb();
 321         pending = timer_pending(&pcpu->cpu_timer);
 322
 323         if (pcpu->target_freq != pcpu->policy->min) {
 324 #ifdef CONFIG_SMP
 325                 /*
 326                  * Entering idle while not at lowest speed.  On some
 327                  * platforms this can hold the other CPU(s) at that speed
 328                  * even though the CPU is idle. Set a timer to re-evaluate
 329                  * speed so this idle CPU doesn't hold the other CPUs above
 330                  * min indefinitely.  This should probably be a quirk of
 331                  * the CPUFreq driver.
 332                  */
 333                 if (!pending) {
 334                         pcpu->time_in_idle = get_cpu_idle_time_us(
 335                                 smp_processor_id(), &pcpu->idle_exit_time);
 336                         pcpu->timer_idlecancel = 0;
 337                         mod_timer(&pcpu->cpu_timer,
 338                                   jiffies + usecs_to_jiffies(timer_rate));
 339                 }
 340 #endif
 341         } else {
 342                 /*
 343                  * If at min speed and entering idle after load has
 344                  * already been evaluated, and a timer has been set just in
 345                  * case the CPU suddenly goes busy, cancel that timer.  The
 346                  * CPU didn't go busy; we'll recheck things upon idle exit.
 347                  */
 348                 if (pending && pcpu->timer_idlecancel) {
 349                         del_timer(&pcpu->cpu_timer);
 350                         /*
 351                          * Ensure last timer run time is after current idle
 352                          * sample start time, so next idle exit will always
 353                          * start a new idle sampling period.
 354                          */
 355                         pcpu->idle_exit_time = 0;
 356                         pcpu->timer_idlecancel = 0;
 357                 }
 358         }
 359
 360 }
 361
 362 static void cpufreq_interactive_idle_end(void)
 363 {
 364         struct cpufreq_interactive_cpuinfo *pcpu =
 365                 &per_cpu(cpuinfo, smp_processor_id());
 366
 367         pcpu->idling = 0;
 368         smp_wmb();
 369
 370         /*
 371          * Arm the timer for 1-2 ticks later if not already, and if the timer
 372          * function has already processed the previous load sampling
 373          * interval.  (If the timer is not pending but has not processed
 374          * the previous interval, it is probably racing with us on another
 375          * CPU.  Let it compute load based on the previous sample and then
 376          * re-arm the timer for another interval when it's done, rather
 377          * than updating the interval start time to be "now", which doesn't
 378          * give the timer function enough time to make a decision on this
 379          * run.)
 380          */
 381         if (timer_pending(&pcpu->cpu_timer) == 0 &&
 382             pcpu->timer_run_time >= pcpu->idle_exit_time &&
 383             pcpu->governor_enabled) {
 384                 pcpu->time_in_idle =
 385                         get_cpu_idle_time_us(smp_processor_id(),
 386                                              &pcpu->idle_exit_time);
 387                 pcpu->timer_idlecancel = 0;
 388                 mod_timer(&pcpu->cpu_timer,
 389                           jiffies + usecs_to_jiffies(timer_rate));
 390         }
 391
 392 }
 393
 394 static int cpufreq_interactive_up_task(void *data)
 395 {
 396         unsigned int cpu;
 397         cpumask_t tmp_mask;
 398         unsigned long flags;
 399         struct cpufreq_interactive_cpuinfo *pcpu;
 400
 401         while (1) {
 402                 set_current_state(TASK_INTERRUPTIBLE);
 403                 spin_lock_irqsave(&up_cpumask_lock, flags);
 404
 405                 if (cpumask_empty(&up_cpumask)) {
 406                         spin_unlock_irqrestore(&up_cpumask_lock, flags);
 407                         schedule();
 408
 409                         if (kthread_should_stop())
 410                                 break;
 411
 412                         spin_lock_irqsave(&up_cpumask_lock, flags);
 413                 }
 414
 415                 set_current_state(TASK_RUNNING);
 416                 tmp_mask = up_cpumask;
 417                 cpumask_clear(&up_cpumask);
 418                 spin_unlock_irqrestore(&up_cpumask_lock, flags);
 419
 420                 for_each_cpu(cpu, &tmp_mask) {
 421                         unsigned int j;
 422                         unsigned int max_freq = 0;
 423
 424                         pcpu = &per_cpu(cpuinfo, cpu);
 425                         smp_rmb();
 426
 427                         if (!pcpu->governor_enabled)
 428                                 continue;
 429
 430                         mutex_lock(&set_speed_lock);
 431
 432                         for_each_cpu(j, pcpu->policy->cpus) {
 433                                 struct cpufreq_interactive_cpuinfo *pjcpu =
 434                                         &per_cpu(cpuinfo, j);
 435
 436                                 if (pjcpu->target_freq > max_freq)
 437                                         max_freq = pjcpu->target_freq;
 438                         }
 439
 440                         if (max_freq != pcpu->policy->cur)
 441                                 __cpufreq_driver_target(pcpu->policy,
 442                                                         max_freq,
 443                                                         CPUFREQ_RELATION_H);
 444                         mutex_unlock(&set_speed_lock);
 445                         trace_cpufreq_interactive_up(cpu, pcpu->target_freq,
 446                                                      pcpu->policy->cur);
 447                 }
 448         }
 449
 450         return 0;
 451 }
 452
 453 static void cpufreq_interactive_freq_down(struct work_struct *work)
 454 {
 455         unsigned int cpu;
 456         cpumask_t tmp_mask;
 457         unsigned long flags;
 458         struct cpufreq_interactive_cpuinfo *pcpu;
 459
 460         spin_lock_irqsave(&down_cpumask_lock, flags);
 461         tmp_mask = down_cpumask;
 462         cpumask_clear(&down_cpumask);
 463         spin_unlock_irqrestore(&down_cpumask_lock, flags);
 464
 465         for_each_cpu(cpu, &tmp_mask) {
 466                 unsigned int j;
 467                 unsigned int max_freq = 0;
 468
 469                 pcpu = &per_cpu(cpuinfo, cpu);
 470                 smp_rmb();
 471
 472                 if (!pcpu->governor_enabled)
 473                         continue;
 474
 475                 mutex_lock(&set_speed_lock);
 476
 477                 for_each_cpu(j, pcpu->policy->cpus) {
 478                         struct cpufreq_interactive_cpuinfo *pjcpu =
 479                                 &per_cpu(cpuinfo, j);
 480
 481                         if (pjcpu->target_freq > max_freq)
 482                                 max_freq = pjcpu->target_freq;
 483                 }
 484
 485                 if (max_freq != pcpu->policy->cur)
 486                         __cpufreq_driver_target(pcpu->policy, max_freq,
 487                                                 CPUFREQ_RELATION_H);
 488
 489                 mutex_unlock(&set_speed_lock);
 490                 trace_cpufreq_interactive_down(cpu, pcpu->target_freq,
 491                                                pcpu->policy->cur);
 492         }
 493 }
 494
 495 static void cpufreq_interactive_boost(void)
 496 {
 497         int i;
 498         int anyboost = 0;
 499         unsigned long flags;
 500         struct cpufreq_interactive_cpuinfo *pcpu;
 501
 502         spin_lock_irqsave(&up_cpumask_lock, flags);
 503
 504         for_each_online_cpu(i) {
 505                 pcpu = &per_cpu(cpuinfo, i);
 506
 507                 if (pcpu->target_freq < hispeed_freq) {
 508                         pcpu->target_freq = hispeed_freq;
 509                         cpumask_set_cpu(i, &up_cpumask);
 510                         pcpu->target_set_time_in_idle =
 511                                 get_cpu_idle_time_us(i, &pcpu->target_set_time);
 512                         pcpu->hispeed_validate_time = pcpu->target_set_time;
 513                         anyboost = 1;
 514                 }
 515
 516                 /*
 517                  * Set floor freq and (re)start timer for when last
 518                  * validated.
 519                  */
 520
 521                 pcpu->floor_freq = hispeed_freq;
 522                 pcpu->floor_validate_time = ktime_to_us(ktime_get());
 523         }
 524
 525         spin_unlock_irqrestore(&up_cpumask_lock, flags);
 526
 527         if (anyboost)
 528                 wake_up_process(up_task);
 529 }
 530
 531 /*
 532  * Pulsed boost on input event raises CPUs to hispeed_freq and lets
 533  * usual algorithm of min_sample_time  decide when to allow speed
 534  * to drop.
 535  */
 536
 537 static void cpufreq_interactive_input_event(struct input_handle *handle,
 538                                             unsigned int type,
 539                                             unsigned int code, int value)
 540 {
 541         if (input_boost_val && type == EV_SYN && code == SYN_REPORT) {
 542                 trace_cpufreq_interactive_boost("input");
 543                 cpufreq_interactive_boost();
 544         }
 545 }
 546
 547 static void cpufreq_interactive_input_open(struct work_struct *w)
 548 {
 549         struct cpufreq_interactive_inputopen *io =
 550                 container_of(w, struct cpufreq_interactive_inputopen,
 551                              inputopen_work);
 552         int error;
 553
 554         error = input_open_device(io->handle);
 555         if (error)
 556                 input_unregister_handle(io->handle);
 557 }
 558
 559 static int cpufreq_interactive_input_connect(struct input_handler *handler,
 560                                              struct input_dev *dev,
 561                                              const struct input_device_id *id)
 562 {
 563         struct input_handle *handle;
 564         int error;
 565
 566         pr_info("%s: connect to %s\n", __func__, dev->name);
 567         handle = kzalloc(sizeof(struct input_handle), GFP_KERNEL);
 568         if (!handle)
 569                 return -ENOMEM;
 570
 571         handle->dev = dev;
 572         handle->handler = handler;
 573         handle->name = "cpufreq_interactive";
 574
 575         error = input_register_handle(handle);
 576         if (error)
 577                 goto err;
 578
 579         inputopen.handle = handle;
 580         queue_work(down_wq, &inputopen.inputopen_work);
 581         return 0;
 582 err:
 583         kfree(handle);
 584         return error;
 585 }
 586
 587 static void cpufreq_interactive_input_disconnect(struct input_handle *handle)
 588 {
 589         input_close_device(handle);
 590         input_unregister_handle(handle);
 591         kfree(handle);
 592 }
 593
 594 static const struct input_device_id cpufreq_interactive_ids[] = {
 595         {
 596                 .flags = INPUT_DEVICE_ID_MATCH_EVBIT |
 597                          INPUT_DEVICE_ID_MATCH_ABSBIT,
 598                 .evbit = { BIT_MASK(EV_ABS) },
 599                 .absbit = { [BIT_WORD(ABS_MT_POSITION_X)] =
 600                             BIT_MASK(ABS_MT_POSITION_X) |
 601                             BIT_MASK(ABS_MT_POSITION_Y) },
 602         }, /* multi-touch touchscreen */
 603         {
 604                 .flags = INPUT_DEVICE_ID_MATCH_KEYBIT |
 605                          INPUT_DEVICE_ID_MATCH_ABSBIT,
 606                 .keybit = { [BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH) },
 607                 .absbit = { [BIT_WORD(ABS_X)] =
 608                             BIT_MASK(ABS_X) | BIT_MASK(ABS_Y) },
 609         }, /* touchpad */
 610         { },
 611 };
 612
 613 static struct input_handler cpufreq_interactive_input_handler = {
 614         .event          = cpufreq_interactive_input_event,
 615         .connect        = cpufreq_interactive_input_connect,
 616         .disconnect     = cpufreq_interactive_input_disconnect,
 617         .name           = "cpufreq_interactive",
 618         .id_table       = cpufreq_interactive_ids,
 619 };
 620
 621 static ssize_t show_hispeed_freq(struct kobject *kobj,
 622                                  struct attribute *attr, char *buf)
 623 {
 624         return sprintf(buf, "%llu\n", hispeed_freq);
 625 }
 626
 627 static ssize_t store_hispeed_freq(struct kobject *kobj,
 628                                   struct attribute *attr, const char *buf,
 629                                   size_t count)
 630 {
 631         int ret;
 632         u64 val;
 633
 634         ret = strict_strtoull(buf, 0, &val);
 635         if (ret < 0)
 636                 return ret;
 637         hispeed_freq = val;
 638         return count;
 639 }
 640
 641 static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644,
 642                 show_hispeed_freq, store_hispeed_freq);
 643
 644
 645 static ssize_t show_go_hispeed_load(struct kobject *kobj,
 646                                      struct attribute *attr, char *buf)
 647 {
 648         return sprintf(buf, "%lu\n", go_hispeed_load);
 649 }
 650
 651 static ssize_t store_go_hispeed_load(struct kobject *kobj,
 652                         struct attribute *attr, const char *buf, size_t count)
 653 {
 654         int ret;
 655         unsigned long val;
 656
 657         ret = strict_strtoul(buf, 0, &val);
 658         if (ret < 0)
 659                 return ret;
 660         go_hispeed_load = val;
 661         return count;
 662 }
 663
 664 static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644,
 665                 show_go_hispeed_load, store_go_hispeed_load);
 666
 667 static ssize_t show_min_sample_time(struct kobject *kobj,
 668                                 struct attribute *attr, char *buf)
 669 {
 670         return sprintf(buf, "%lu\n", min_sample_time);
 671 }
 672
 673 static ssize_t store_min_sample_time(struct kobject *kobj,
 674                         struct attribute *attr, const char *buf, size_t count)
 675 {
 676         int ret;
 677         unsigned long val;
 678
 679         ret = strict_strtoul(buf, 0, &val);
 680         if (ret < 0)
 681                 return ret;
 682         min_sample_time = val;
 683         return count;
 684 }
 685
 686 static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
 687                 show_min_sample_time, store_min_sample_time);
 688
 689 static ssize_t show_above_hispeed_delay(struct kobject *kobj,
 690                                         struct attribute *attr, char *buf)
 691 {
 692         return sprintf(buf, "%lu\n", above_hispeed_delay_val);
 693 }
 694
 695 static ssize_t store_above_hispeed_delay(struct kobject *kobj,
 696                                          struct attribute *attr,
 697                                          const char *buf, size_t count)
 698 {
 699         int ret;
 700         unsigned long val;
 701
 702         ret = strict_strtoul(buf, 0, &val);
 703         if (ret < 0)
 704                 return ret;
 705         above_hispeed_delay_val = val;
 706         return count;
 707 }
 708
 709 define_one_global_rw(above_hispeed_delay);
 710
 711 static ssize_t show_timer_rate(struct kobject *kobj,
 712                         struct attribute *attr, char *buf)
 713 {
 714         return sprintf(buf, "%lu\n", timer_rate);
 715 }
 716
 717 static ssize_t store_timer_rate(struct kobject *kobj,
 718                         struct attribute *attr, const char *buf, size_t count)
 719 {
 720         int ret;
 721         unsigned long val;
 722
 723         ret = strict_strtoul(buf, 0, &val);
 724         if (ret < 0)
 725                 return ret;
 726         timer_rate = val;
 727         return count;
 728 }
 729
 730 static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
 731                 show_timer_rate, store_timer_rate);
 732
 733 static ssize_t show_input_boost(struct kobject *kobj, struct attribute *attr,
 734                                 char *buf)
 735 {
 736         return sprintf(buf, "%u\n", input_boost_val);
 737 }
 738
 739 static ssize_t store_input_boost(struct kobject *kobj, struct attribute *attr,
 740                                  const char *buf, size_t count)
 741 {
 742         int ret;
 743         unsigned long val;
 744
 745         ret = strict_strtoul(buf, 0, &val);
 746         if (ret < 0)
 747                 return ret;
 748         input_boost_val = val;
 749         return count;
 750 }
 751
 752 define_one_global_rw(input_boost);
 753
 754 static ssize_t show_boost(struct kobject *kobj, struct attribute *attr,
 755                           char *buf)
 756 {
 757         return sprintf(buf, "%d\n", boost_val);
 758 }
 759
 760 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 761                            const char *buf, size_t count)
 762 {
 763         int ret;
 764         unsigned long val;
 765
 766         ret = kstrtoul(buf, 0, &val);
 767         if (ret < 0)
 768                 return ret;
 769
 770         boost_val = val;
 771
 772         if (boost_val) {
 773                 trace_cpufreq_interactive_boost("on");
 774                 cpufreq_interactive_boost();
 775         } else {
 776                 trace_cpufreq_interactive_unboost("off");
 777         }
 778
 779         return count;
 780 }
 781
 782 define_one_global_rw(boost);
 783
 784 static ssize_t store_boostpulse(struct kobject *kobj, struct attribute *attr,
 785                                 const char *buf, size_t count)
 786 {
 787         int ret;
 788         unsigned long val;
 789
 790         ret = kstrtoul(buf, 0, &val);
 791         if (ret < 0)
 792                 return ret;
 793
 794         trace_cpufreq_interactive_boost("pulse");
 795         cpufreq_interactive_boost();
 796         return count;
 797 }
 798
 799 static struct global_attr boostpulse =
 800         __ATTR(boostpulse, 0200, NULL, store_boostpulse);
 801
 802 static struct attribute *interactive_attributes[] = {
 803         &hispeed_freq_attr.attr,
 804         &go_hispeed_load_attr.attr,
 805         &above_hispeed_delay.attr,
 806         &min_sample_time_attr.attr,
 807         &timer_rate_attr.attr,
 808         &input_boost.attr,
 809         &boost.attr,
 810         &boostpulse.attr,
 811         NULL,
 812 };
 813
 814 static struct attribute_group interactive_attr_group = {
 815         .attrs = interactive_attributes,
 816         .name = "interactive",
 817 };
 818
 819 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 820                 unsigned int event)
 821 {
 822         int rc;
 823         unsigned int j;
 824         struct cpufreq_interactive_cpuinfo *pcpu;
 825         struct cpufreq_frequency_table *freq_table;
 826
 827         switch (event) {
 828         case CPUFREQ_GOV_START:
 829                 if (!cpu_online(policy->cpu))
 830                         return -EINVAL;
 831
 832                 freq_table =
 833                         cpufreq_frequency_get_table(policy->cpu);
 834
 835                 for_each_cpu(j, policy->cpus) {
 836                         pcpu = &per_cpu(cpuinfo, j);
 837                         pcpu->policy = policy;
 838                         pcpu->target_freq = policy->cur;
 839                         pcpu->freq_table = freq_table;
 840                         pcpu->target_set_time_in_idle =
 841                                 get_cpu_idle_time_us(j,
 842                                              &pcpu->target_set_time);
 843                         pcpu->floor_freq = pcpu->target_freq;
 844                         pcpu->floor_validate_time =
 845                                 pcpu->target_set_time;
 846                         pcpu->hispeed_validate_time =
 847                                 pcpu->target_set_time;
 848                         pcpu->governor_enabled = 1;
 849                         smp_wmb();
 850                 }
 851
 852                 if (!hispeed_freq)
 853                         hispeed_freq = policy->max;
 854
 855                 /*
 856                  * Do not register the idle hook and create sysfs
 857                  * entries if we have already done so.
 858                  */
 859                 if (atomic_inc_return(&active_count) > 1)
 860                         return 0;
 861
 862                 rc = sysfs_create_group(cpufreq_global_kobject,
 863                                 &interactive_attr_group);
 864                 if (rc)
 865                         return rc;
 866
 867                 rc = input_register_handler(&cpufreq_interactive_input_handler);
 868                 if (rc)
 869                         pr_warn("%s: failed to register input handler\n",
 870                                 __func__);
 871
 872                 break;
 873
 874         case CPUFREQ_GOV_STOP:
 875                 for_each_cpu(j, policy->cpus) {
 876                         pcpu = &per_cpu(cpuinfo, j);
 877                         pcpu->governor_enabled = 0;
 878                         smp_wmb();
 879                         del_timer_sync(&pcpu->cpu_timer);
 880
 881                         /*
 882                          * Reset idle exit time since we may cancel the timer
 883                          * before it can run after the last idle exit time,
 884                          * to avoid tripping the check in idle exit for a timer
 885                          * that is trying to run.
 886                          */
 887                         pcpu->idle_exit_time = 0;
 888                 }
 889
 890                 flush_work(&freq_scale_down_work);
 891                 if (atomic_dec_return(&active_count) > 0)
 892                         return 0;
 893
 894                 input_unregister_handler(&cpufreq_interactive_input_handler);
 895                 sysfs_remove_group(cpufreq_global_kobject,
 896                                 &interactive_attr_group);
 897
 898                 break;
 899
 900         case CPUFREQ_GOV_LIMITS:
 901                 if (policy->max < policy->cur)
 902                         __cpufreq_driver_target(policy,
 903                                         policy->max, CPUFREQ_RELATION_H);
 904                 else if (policy->min > policy->cur)
 905                         __cpufreq_driver_target(policy,
 906                                         policy->min, CPUFREQ_RELATION_L);
 907                 break;
 908         }
 909         return 0;
 910 }
 911
 912 static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
 913                                              unsigned long val,
 914                                              void *data)
 915 {
 916         switch (val) {
 917         case IDLE_START:
 918                 cpufreq_interactive_idle_start();
 919                 break;
 920         case IDLE_END:
 921                 cpufreq_interactive_idle_end();
 922                 break;
 923         }
 924
 925         return 0;
 926 }
 927
 928 static struct notifier_block cpufreq_interactive_idle_nb = {
 929         .notifier_call = cpufreq_interactive_idle_notifier,
 930 };
 931
 932 static int __init cpufreq_interactive_init(void)
 933 {
 934         unsigned int i;
 935         struct cpufreq_interactive_cpuinfo *pcpu;
 936         struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 937
 938         go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
 939         min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
 940         above_hispeed_delay_val = DEFAULT_ABOVE_HISPEED_DELAY;
 941         timer_rate = DEFAULT_TIMER_RATE;
 942
 943         /* Initalize per-cpu timers */
 944         for_each_possible_cpu(i) {
 945                 pcpu = &per_cpu(cpuinfo, i);
 946                 init_timer(&pcpu->cpu_timer);
 947                 pcpu->cpu_timer.function = cpufreq_interactive_timer;
 948                 pcpu->cpu_timer.data = i;
 949         }
 950
 951         up_task = kthread_create(cpufreq_interactive_up_task, NULL,
 952                                  "kinteractiveup");
 953         if (IS_ERR(up_task))
 954                 return PTR_ERR(up_task);
 955
 956         sched_setscheduler_nocheck(up_task, SCHED_FIFO, &param);
 957         get_task_struct(up_task);
 958
 959         /* No rescuer thread, bind to CPU queuing the work for possibly
 960            warm cache (probably doesn't matter much). */
 961         down_wq = alloc_workqueue("knteractive_down", 0, 1);
 962
 963         if (!down_wq)
 964                 goto err_freeuptask;
 965
 966         INIT_WORK(&freq_scale_down_work,
 967                   cpufreq_interactive_freq_down);
 968
 969         spin_lock_init(&up_cpumask_lock);
 970         spin_lock_init(&down_cpumask_lock);
 971         mutex_init(&set_speed_lock);
 972
 973         idle_notifier_register(&cpufreq_interactive_idle_nb);
 974         INIT_WORK(&inputopen.inputopen_work, cpufreq_interactive_input_open);
 975         return cpufreq_register_governor(&cpufreq_gov_interactive);
 976
 977 err_freeuptask:
 978         put_task_struct(up_task);
 979         return -ENOMEM;
 980 }
 981
 982 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 983 fs_initcall(cpufreq_interactive_init);
 984 #else
 985 module_init(cpufreq_interactive_init);
 986 #endif
 987
 988 static void __exit cpufreq_interactive_exit(void)
 989 {
 990         cpufreq_unregister_governor(&cpufreq_gov_interactive);
 991         kthread_stop(up_task);
 992         put_task_struct(up_task);
 993         destroy_workqueue(down_wq);
 994 }
 995
 996 module_exit(cpufreq_interactive_exit);
 997
 998 MODULE_AUTHOR("Mike Chan <mike@android.com>");
 999 MODULE_DESCRIPTION("'cpufreq_interactive' - A cpufreq governor for "
1000         "Latency sensitive workloads");
1001 MODULE_LICENSE("GPL");