#include <linux/workqueue.h>
#include <linux/kthread.h>
#include <linux/slab.h>
-#include "cpufreq_governor.h"
#define CREATE_TRACE_POINTS
#include <trace/events/cpufreq_interactive.h>
spinlock_t target_freq_lock; /*protects target freq */
unsigned int target_freq;
unsigned int floor_freq;
- unsigned int max_freq;
- u64 floor_validate_time;
- u64 hispeed_validate_time;
+ u64 pol_floor_val_time; /* policy floor_validate_time */
+ u64 loc_floor_val_time; /* per-cpu floor_validate_time */
+ u64 pol_hispeed_val_time; /* policy hispeed_validate_time */
+ u64 loc_hispeed_val_time; /* per-cpu hispeed_validate_time */
struct rw_semaphore enable_sem;
int governor_enabled;
};
pcpu->policy->governor_data;
u64 now;
u64 now_idle;
- u64 delta_idle;
- u64 delta_time;
+ unsigned int delta_idle;
+ unsigned int delta_time;
u64 active_time;
now_idle = get_cpu_idle_time(cpu, &now, tunables->io_is_busy);
- delta_idle = (now_idle - pcpu->time_in_idle);
- delta_time = (now - pcpu->time_in_idle_timestamp);
+ delta_idle = (unsigned int)(now_idle - pcpu->time_in_idle);
+ delta_time = (unsigned int)(now - pcpu->time_in_idle_timestamp);
if (delta_time <= delta_idle)
active_time = 0;
unsigned int loadadjfreq;
unsigned int index;
unsigned long flags;
+ u64 max_fvtime;
if (!down_read_trylock(&pcpu->enable_sem))
return;
spin_lock_irqsave(&pcpu->target_freq_lock, flags);
do_div(cputime_speedadj, delta_time);
loadadjfreq = (unsigned int)cputime_speedadj * 100;
- cpu_load = loadadjfreq / pcpu->target_freq;
+ cpu_load = loadadjfreq / pcpu->policy->cur;
tunables->boosted = tunables->boost_val || now < tunables->boostpulse_endtime;
-#ifdef CONFIG_ARCH_ROCKCHIP
- pcpu->target_freq = pcpu->policy->cur;
- tunables->boosted |= now < tunables->touchboostpulse_endtime;
-#endif
-
if (cpu_load >= tunables->go_hispeed_load || tunables->boosted) {
-#ifdef CONFIG_ARCH_ROCKCHIP
- if (now < tunables->touchboostpulse_endtime) {
- new_freq = choose_freq(pcpu, loadadjfreq);
- if (new_freq < tunables->touchboost_freq)
- new_freq = tunables->touchboost_freq;
- } else
-#endif
- if (pcpu->target_freq < tunables->hispeed_freq) {
+ if (pcpu->policy->cur < tunables->hispeed_freq) {
new_freq = tunables->hispeed_freq;
} else {
new_freq = choose_freq(pcpu, loadadjfreq);
} else {
new_freq = choose_freq(pcpu, loadadjfreq);
if (new_freq > tunables->hispeed_freq &&
- pcpu->target_freq < tunables->hispeed_freq)
+ pcpu->policy->cur < tunables->hispeed_freq)
new_freq = tunables->hispeed_freq;
}
-
- if (pcpu->target_freq >= tunables->hispeed_freq &&
- new_freq > pcpu->target_freq &&
- now - pcpu->hispeed_validate_time <
- freq_to_above_hispeed_delay(tunables, pcpu->target_freq)) {
+#ifdef CONFIG_ARCH_ROCKCHIP
+ if ((now < tunables->touchboostpulse_endtime) &&
+ (new_freq < tunables->touchboost_freq)) {
+ new_freq = tunables->touchboost_freq;
+ }
+#endif
+ if (pcpu->policy->cur >= tunables->hispeed_freq &&
+ new_freq > pcpu->policy->cur &&
+ now - pcpu->pol_hispeed_val_time <
+ freq_to_above_hispeed_delay(tunables, pcpu->policy->cur)) {
trace_cpufreq_interactive_notyet(
data, cpu_load, pcpu->target_freq,
pcpu->policy->cur, new_freq);
goto rearm;
}
- pcpu->hispeed_validate_time = now;
+ pcpu->loc_hispeed_val_time = now;
if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
new_freq, CPUFREQ_RELATION_L,
* Do not scale below floor_freq unless we have been at or above the
* floor frequency for the minimum sample time since last validated.
*/
- if (new_freq < pcpu->floor_freq) {
- if (now - pcpu->floor_validate_time <
- tunables->min_sample_time) {
+ max_fvtime = max(pcpu->pol_floor_val_time, pcpu->loc_floor_val_time);
+ if (new_freq < pcpu->floor_freq &&
+ pcpu->target_freq >= pcpu->policy->cur) {
+ if (now - max_fvtime < tunables->min_sample_time) {
trace_cpufreq_interactive_notyet(
data, cpu_load, pcpu->target_freq,
pcpu->policy->cur, new_freq);
if (!tunables->boosted || new_freq > tunables->hispeed_freq) {
pcpu->floor_freq = new_freq;
- pcpu->floor_validate_time = now;
+ if (pcpu->target_freq >= pcpu->policy->cur ||
+ new_freq >= pcpu->policy->cur)
+ pcpu->loc_floor_val_time = now;
}
if (pcpu->target_freq == new_freq &&
data, cpu_load, pcpu->target_freq,
pcpu->policy->cur, new_freq);
spin_unlock_irqrestore(&pcpu->target_freq_lock, flags);
- goto rearm_if_notmax;
+ goto rearm;
}
trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
wake_up_process(speedchange_task);
-rearm_if_notmax:
- /*
- * Already set max speed and don't see a need to change that,
- * wait until next idle to re-evaluate, don't need timer.
- */
- if (pcpu->target_freq == pcpu->policy->max)
- goto exit;
-
rearm:
if (!timer_pending(&pcpu->cpu_timer))
cpufreq_interactive_timer_resched(pcpu);
return;
}
-static void cpufreq_interactive_idle_start(void)
-{
- struct cpufreq_interactive_cpuinfo *pcpu =
- &per_cpu(cpuinfo, smp_processor_id());
- int pending;
-
- if (!down_read_trylock(&pcpu->enable_sem))
- return;
- if (!pcpu->governor_enabled) {
- up_read(&pcpu->enable_sem);
- return;
- }
-
- pending = timer_pending(&pcpu->cpu_timer);
-
- if (pcpu->target_freq != pcpu->policy->min) {
- /*
- * Entering idle while not at lowest speed. On some
- * platforms this can hold the other CPU(s) at that speed
- * even though the CPU is idle. Set a timer to re-evaluate
- * speed so this idle CPU doesn't hold the other CPUs above
- * min indefinitely. This should probably be a quirk of
- * the CPUFreq driver.
- */
- if (!pending)
- cpufreq_interactive_timer_resched(pcpu);
- }
-
- up_read(&pcpu->enable_sem);
-}
-
static void cpufreq_interactive_idle_end(void)
{
struct cpufreq_interactive_cpuinfo *pcpu =
up_read(&pcpu->enable_sem);
}
+static void cpufreq_interactive_get_policy_info(struct cpufreq_policy *policy,
+ unsigned int *pmax_freq,
+ u64 *phvt, u64 *pfvt)
+{
+ struct cpufreq_interactive_cpuinfo *pcpu;
+ unsigned int max_freq = 0;
+ u64 hvt = ~0ULL, fvt = 0;
+ unsigned int i;
+
+ for_each_cpu(i, policy->cpus) {
+ pcpu = &per_cpu(cpuinfo, i);
+
+ fvt = max(fvt, pcpu->loc_floor_val_time);
+ if (pcpu->target_freq > max_freq) {
+ max_freq = pcpu->target_freq;
+ hvt = pcpu->loc_hispeed_val_time;
+ } else if (pcpu->target_freq == max_freq) {
+ hvt = min(hvt, pcpu->loc_hispeed_val_time);
+ }
+ }
+
+ *pmax_freq = max_freq;
+ *phvt = hvt;
+ *pfvt = fvt;
+}
+
+static void cpufreq_interactive_adjust_cpu(unsigned int cpu,
+ struct cpufreq_policy *policy)
+{
+ struct cpufreq_interactive_cpuinfo *pcpu;
+ u64 hvt, fvt;
+ unsigned int max_freq;
+ int i;
+
+ cpufreq_interactive_get_policy_info(policy, &max_freq, &hvt, &fvt);
+
+ for_each_cpu(i, policy->cpus) {
+ pcpu = &per_cpu(cpuinfo, i);
+ pcpu->pol_floor_val_time = fvt;
+ }
+
+ if (max_freq != policy->cur) {
+ __cpufreq_driver_target(policy, max_freq, CPUFREQ_RELATION_H);
+ for_each_cpu(i, policy->cpus) {
+ pcpu = &per_cpu(cpuinfo, i);
+ pcpu->pol_hispeed_val_time = hvt;
+ }
+ }
+
+ trace_cpufreq_interactive_setspeed(cpu, max_freq, policy->cur);
+}
+
static int cpufreq_interactive_speedchange_task(void *data)
{
unsigned int cpu;
spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
for_each_cpu(cpu, &tmp_mask) {
- unsigned int j;
- unsigned int max_freq = 0;
-
pcpu = &per_cpu(cpuinfo, cpu);
- if (!down_read_trylock(&pcpu->enable_sem))
- continue;
- if (!pcpu->governor_enabled) {
- up_read(&pcpu->enable_sem);
- continue;
- }
- for_each_cpu(j, pcpu->policy->cpus) {
- struct cpufreq_interactive_cpuinfo *pjcpu =
- &per_cpu(cpuinfo, j);
+ down_write(&pcpu->policy->rwsem);
- if (pjcpu->target_freq > max_freq)
- max_freq = pjcpu->target_freq;
+ if (likely(down_read_trylock(&pcpu->enable_sem))) {
+ if (likely(pcpu->governor_enabled))
+ cpufreq_interactive_adjust_cpu(cpu,
+ pcpu->policy);
+ up_read(&pcpu->enable_sem);
}
- if (max_freq != pcpu->policy->cur)
- __cpufreq_driver_target(pcpu->policy,
- max_freq,
- CPUFREQ_RELATION_H);
- trace_cpufreq_interactive_setspeed(cpu,
- pcpu->target_freq,
- pcpu->policy->cur);
-
- up_read(&pcpu->enable_sem);
+ up_write(&pcpu->policy->rwsem);
}
}
for_each_online_cpu(i) {
pcpu = &per_cpu(cpuinfo, i);
- if (tunables != pcpu->policy->governor_data)
+
+ if (!down_read_trylock(&pcpu->enable_sem))
+ continue;
+
+ if (!pcpu->governor_enabled) {
+ up_read(&pcpu->enable_sem);
continue;
+ }
+
+ if (tunables != pcpu->policy->governor_data) {
+ up_read(&pcpu->enable_sem);
+ continue;
+ }
spin_lock_irqsave(&pcpu->target_freq_lock, flags[1]);
if (pcpu->target_freq < tunables->hispeed_freq) {
pcpu->target_freq = tunables->hispeed_freq;
cpumask_set_cpu(i, &speedchange_cpumask);
- pcpu->hispeed_validate_time =
+ pcpu->pol_hispeed_val_time =
ktime_to_us(ktime_get());
anyboost = 1;
}
-
- /*
- * Set floor freq and (re)start timer for when last
- * validated.
- */
-
- pcpu->floor_freq = tunables->hispeed_freq;
- pcpu->floor_validate_time = ktime_to_us(ktime_get());
spin_unlock_irqrestore(&pcpu->target_freq_lock, flags[1]);
+
+ up_read(&pcpu->enable_sem);
}
spin_unlock_irqrestore(&speedchange_cpumask_lock, flags[0]);
int ret;
long unsigned int val;
- ret = strict_strtoul(buf, 0, &val);
+ ret = kstrtoul(buf, 0, &val);
if (ret < 0)
return ret;
tunables->hispeed_freq = val;
int ret;
unsigned long val;
- ret = strict_strtoul(buf, 0, &val);
+ ret = kstrtoul(buf, 0, &val);
if (ret < 0)
return ret;
tunables->go_hispeed_load = val;
int ret;
unsigned long val;
- ret = strict_strtoul(buf, 0, &val);
+ ret = kstrtoul(buf, 0, &val);
if (ret < 0)
return ret;
tunables->min_sample_time = val;
const char *buf, size_t count)
{
int ret;
- unsigned long val;
+ unsigned long val, val_round;
- ret = strict_strtoul(buf, 0, &val);
+ ret = kstrtoul(buf, 0, &val);
if (ret < 0)
return ret;
- tunables->timer_rate = val;
+
+ val_round = jiffies_to_usecs(usecs_to_jiffies(val));
+ if (val != val_round)
+ pr_warn("timer_rate not aligned to jiffy. Rounded up to %lu\n",
+ val_round);
+
+ tunables->timer_rate = val_round;
return count;
}
unsigned long val,
void *data)
{
- switch (val) {
- case IDLE_START:
- cpufreq_interactive_idle_start();
- break;
- case IDLE_END:
+ if (val == IDLE_END)
cpufreq_interactive_idle_end();
- break;
- }
return 0;
}
};
#ifdef CONFIG_ARCH_ROCKCHIP
-static void cpufreq_interactive_input_event(struct input_handle *handle, unsigned int type,
- unsigned int code, int value)
+static void cpufreq_interactive_input_event(struct input_handle *handle,
+ unsigned int type,
+ unsigned int code,
+ int value)
{
u64 now, endtime;
int i;
struct cpufreq_interactive_cpuinfo *pcpu;
struct cpufreq_interactive_tunables *tunables;
- if (type != EV_ABS)
+ if ((type != EV_ABS) && (type != EV_KEY))
return;
trace_cpufreq_interactive_boost("touch");
now = ktime_to_us(ktime_get());
for_each_online_cpu(i) {
pcpu = &per_cpu(cpuinfo, i);
- tunables = pcpu->policy->governor_data;
+ if (have_governor_per_policy())
+ tunables = pcpu->policy->governor_data;
+ else
+ tunables = common_tunables;
+ if (!tunables)
+ continue;
endtime = now + tunables->touchboostpulse_duration_val;
- if (endtime < (tunables->touchboostpulse_endtime + 10 * USEC_PER_MSEC))
- break;
+ if (endtime < (tunables->touchboostpulse_endtime +
+ 10 * USEC_PER_MSEC))
+ continue;
tunables->touchboostpulse_endtime = endtime;
spin_lock_irqsave(&pcpu->target_freq_lock, flags[1]);
if (pcpu->target_freq < tunables->touchboost_freq) {
pcpu->target_freq = tunables->touchboost_freq;
cpumask_set_cpu(i, &speedchange_cpumask);
- pcpu->hispeed_validate_time =
+ pcpu->loc_hispeed_val_time =
ktime_to_us(ktime_get());
anyboost = 1;
}
pcpu->floor_freq = tunables->touchboost_freq;
- pcpu->floor_validate_time = ktime_to_us(ktime_get());
+ pcpu->loc_floor_val_time = ktime_to_us(ktime_get());
spin_unlock_irqrestore(&pcpu->target_freq_lock, flags[1]);
}
spin_unlock_irqrestore(&speedchange_cpumask_lock, flags[0]);
+
if (anyboost)
wake_up_process(speedchange_task);
}
static int cpufreq_interactive_input_connect(struct input_handler *handler,
- struct input_dev *dev, const struct input_device_id *id)
+ struct input_dev *dev,
+ const struct input_device_id *id)
{
struct input_handle *handle;
int error;
- handle = kzalloc(sizeof(struct input_handle), GFP_KERNEL);
+ handle = kzalloc(sizeof(*handle), GFP_KERNEL);
if (!handle)
return -ENOMEM;
.absbit = { [BIT_WORD(ABS_X)] =
BIT_MASK(ABS_X) | BIT_MASK(ABS_Y) },
},
+ {
+ .flags = INPUT_DEVICE_ID_MATCH_EVBIT,
+ .evbit = { BIT_MASK(EV_KEY) },
+ },
{ },
};
.name = "cpufreq_interactive",
.id_table = cpufreq_interactive_ids,
};
+
+static void rockchip_cpufreq_policy_init(struct cpufreq_policy *policy)
+{
+ struct cpufreq_interactive_tunables *tunables = policy->governor_data;
+
+ tunables->min_sample_time = 40 * USEC_PER_MSEC;
+ tunables->boostpulse_duration_val = 40 * USEC_PER_MSEC;
+ if (policy->cpu == 0) {
+ tunables->hispeed_freq = 1008000;
+ tunables->touchboostpulse_duration_val = 500 * USEC_PER_MSEC;
+ tunables->touchboost_freq = 1200000;
+ } else {
+ tunables->hispeed_freq = 816000;
+ }
+}
#endif
static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
spin_lock_init(&tunables->target_loads_lock);
spin_lock_init(&tunables->above_hispeed_delay_lock);
-#ifdef CONFIG_ARCH_ROCKCHIP
- {
- unsigned int index;
- freq_table = cpufreq_frequency_get_table(policy->cpu);
- tunables->hispeed_freq = policy->max;
- if (policy->min < 600000)
- tunables->hispeed_freq = 600000;
- else if (cpufreq_frequency_table_target(policy, freq_table, policy->min + 1, CPUFREQ_RELATION_L, &index) == 0)
- tunables->hispeed_freq = freq_table[index].frequency;
- tunables->timer_slack_val = 20 * USEC_PER_MSEC;
- tunables->min_sample_time = 40 * USEC_PER_MSEC;
- store_above_hispeed_delay(tunables, "20000 1000000:80000 1200000:100000 1700000:20000", 0);
- store_target_loads(tunables, "70 600000:70 800000:75 1500000:80 1700000:90", 0);
- tunables->boostpulse_duration_val = 40 * USEC_PER_MSEC;
- tunables->touchboostpulse_duration_val = 500 * USEC_PER_MSEC;
- tunables->touchboost_freq = 1200000;
- }
-#endif
-
policy->governor_data = tunables;
- if (!have_governor_per_policy())
+ if (!have_governor_per_policy()) {
common_tunables = tunables;
+ }
+
+#ifdef CONFIG_ARCH_ROCKCHIP
+ rockchip_cpufreq_policy_init(policy);
+#endif
rc = sysfs_create_group(get_governor_parent_kobj(policy),
get_sysfs_attr());
if (rc) {
kfree(tunables);
policy->governor_data = NULL;
- if (!have_governor_per_policy())
+ if (!have_governor_per_policy()) {
common_tunables = NULL;
+ }
return rc;
}
#ifdef CONFIG_ARCH_ROCKCHIP
rc = input_register_handler(&cpufreq_interactive_input_handler);
#endif
+
}
break;
sysfs_remove_group(get_governor_parent_kobj(policy),
get_sysfs_attr());
+
kfree(tunables);
common_tunables = NULL;
}
pcpu->target_freq = policy->cur;
pcpu->freq_table = freq_table;
pcpu->floor_freq = pcpu->target_freq;
- pcpu->floor_validate_time =
+ pcpu->pol_floor_val_time =
ktime_to_us(ktime_get());
- pcpu->hispeed_validate_time =
- pcpu->floor_validate_time;
- pcpu->max_freq = policy->max;
+ pcpu->loc_floor_val_time = pcpu->pol_floor_val_time;
+ pcpu->pol_hispeed_val_time = pcpu->pol_floor_val_time;
+ pcpu->loc_hispeed_val_time = pcpu->pol_floor_val_time;
down_write(&pcpu->enable_sem);
del_timer_sync(&pcpu->cpu_timer);
del_timer_sync(&pcpu->cpu_slack_timer);
spin_unlock_irqrestore(&pcpu->target_freq_lock, flags);
up_read(&pcpu->enable_sem);
-
- /* Reschedule timer only if policy->max is raised.
- * Delete the timers, else the timer callback may
- * return without re-arm the timer when failed
- * acquire the semaphore. This race may cause timer
- * stopped unexpectedly.
- */
-
- if (policy->max > pcpu->max_freq) {
- down_write(&pcpu->enable_sem);
- del_timer_sync(&pcpu->cpu_timer);
- del_timer_sync(&pcpu->cpu_slack_timer);
- cpufreq_interactive_timer_start(tunables, j);
- up_write(&pcpu->enable_sem);
- }
-
- pcpu->max_freq = policy->max;
}
break;
}