X-Git-Url: http://plrg.eecs.uci.edu/git/?p=firefly-linux-kernel-4.4.55.git;a=blobdiff_plain;f=drivers%2Fcpufreq%2Fcpufreq_interactive.c;h=d9927f92ea8b2933ca7cbe35a4ef69011376265d;hp=9dd4d38f2c9e2379d8d4565223e6afd4b148dfe0;hb=0d162fd338537952190c6447a42c8331516ebdc6;hpb=75f9b06a38c833ad66dd41e8e066aa0012b2d421 diff --git a/drivers/cpufreq/cpufreq_interactive.c b/drivers/cpufreq/cpufreq_interactive.c index 9dd4d38f2c9e..d9927f92ea8b 100644 --- a/drivers/cpufreq/cpufreq_interactive.c +++ b/drivers/cpufreq/cpufreq_interactive.c @@ -19,6 +19,9 @@ #include #include #include +#ifdef CONFIG_ARCH_ROCKCHIP +#include +#endif #include #include #include @@ -30,14 +33,10 @@ #include #include #include -#include -#include #define CREATE_TRACE_POINTS #include -static int active_count; - struct cpufreq_interactive_cpuinfo { struct timer_list cpu_timer; struct timer_list cpu_slack_timer; @@ -48,10 +47,13 @@ struct cpufreq_interactive_cpuinfo { u64 cputime_speedadj_timestamp; struct cpufreq_policy *policy; struct cpufreq_frequency_table *freq_table; + spinlock_t target_freq_lock; /*protects target freq */ unsigned int target_freq; unsigned int floor_freq; - u64 floor_validate_time; - u64 hispeed_validate_time; + u64 pol_floor_val_time; /* policy floor_validate_time */ + u64 loc_floor_val_time; /* per-cpu floor_validate_time */ + u64 pol_hispeed_val_time; /* policy hispeed_validate_time */ + u64 loc_hispeed_val_time; /* per-cpu hispeed_validate_time */ struct rw_semaphore enable_sem; int governor_enabled; }; @@ -64,159 +66,163 @@ static cpumask_t speedchange_cpumask; static spinlock_t speedchange_cpumask_lock; static struct mutex gov_lock; -/* Hi speed to bump to from lo speed when load burst (default max) */ -static unsigned int hispeed_freq; - -/* Go to hi speed when CPU load at or above this value. */ -#define DEFAULT_GO_HISPEED_LOAD 99 -static unsigned long go_hispeed_load = DEFAULT_GO_HISPEED_LOAD; - /* Target load. Lower values result in higher CPU speeds. */ #define DEFAULT_TARGET_LOAD 90 static unsigned int default_target_loads[] = {DEFAULT_TARGET_LOAD}; -static spinlock_t target_loads_lock; -static unsigned int *target_loads = default_target_loads; -static int ntarget_loads = ARRAY_SIZE(default_target_loads); -/* - * The minimum amount of time to spend at a frequency before we can ramp down. - */ -#define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC) -static unsigned long min_sample_time = DEFAULT_MIN_SAMPLE_TIME; - -/* - * The sample rate of the timer used to increase frequency - */ #define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC) -static unsigned long timer_rate = DEFAULT_TIMER_RATE; - -/* - * Wait this long before raising speed above hispeed, by default a single - * timer interval. - */ #define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE static unsigned int default_above_hispeed_delay[] = { DEFAULT_ABOVE_HISPEED_DELAY }; -static spinlock_t above_hispeed_delay_lock; -static unsigned int *above_hispeed_delay = default_above_hispeed_delay; -static int nabove_hispeed_delay = ARRAY_SIZE(default_above_hispeed_delay); - -/* Non-zero means indefinite speed boost active */ -static int boost_val; -/* Duration of a boot pulse in usecs */ -static int boostpulse_duration_val = DEFAULT_MIN_SAMPLE_TIME; -/* End time of boost pulse in ktime converted to usecs */ -static u64 boostpulse_endtime; - -/* - * Max additional time to wait in idle, beyond timer_rate, at speeds above - * minimum before wakeup to reduce speed, or -1 if unnecessary. - */ -#define DEFAULT_TIMER_SLACK (4 * DEFAULT_TIMER_RATE) -static int timer_slack_val = DEFAULT_TIMER_SLACK; -static bool io_is_busy; - -static int cpufreq_governor_interactive(struct cpufreq_policy *policy, - unsigned int event); - -#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE -static +struct cpufreq_interactive_tunables { + int usage_count; + /* Hi speed to bump to from lo speed when load burst (default max) */ + unsigned int hispeed_freq; + /* Go to hi speed when CPU load at or above this value. */ +#define DEFAULT_GO_HISPEED_LOAD 99 + unsigned long go_hispeed_load; + /* Target load. Lower values result in higher CPU speeds. */ + spinlock_t target_loads_lock; + unsigned int *target_loads; + int ntarget_loads; + /* + * The minimum amount of time to spend at a frequency before we can ramp + * down. + */ +#define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC) + unsigned long min_sample_time; + /* + * The sample rate of the timer used to increase frequency + */ + unsigned long timer_rate; + /* + * Wait this long before raising speed above hispeed, by default a + * single timer interval. + */ + spinlock_t above_hispeed_delay_lock; + unsigned int *above_hispeed_delay; + int nabove_hispeed_delay; + /* Non-zero means indefinite speed boost active */ + int boost_val; + /* Duration of a boot pulse in usecs */ + int boostpulse_duration_val; + /* End time of boost pulse in ktime converted to usecs */ + u64 boostpulse_endtime; +#ifdef CONFIG_ARCH_ROCKCHIP + /* Frequency to which a touch boost takes the cpus to */ + unsigned long touchboost_freq; + /* Duration of a touchboost pulse in usecs */ + int touchboostpulse_duration_val; + /* End time of touchboost pulse in ktime converted to usecs */ + u64 touchboostpulse_endtime; #endif -struct cpufreq_governor cpufreq_gov_interactive = { - .name = "interactive", - .governor = cpufreq_governor_interactive, - .max_transition_latency = 10000000, - .owner = THIS_MODULE, + bool boosted; + /* + * Max additional time to wait in idle, beyond timer_rate, at speeds + * above minimum before wakeup to reduce speed, or -1 if unnecessary. + */ +#define DEFAULT_TIMER_SLACK (4 * DEFAULT_TIMER_RATE) + int timer_slack_val; + bool io_is_busy; }; -static inline cputime64_t get_cpu_idle_time_jiffy(unsigned int cpu, - cputime64_t *wall) -{ - u64 idle_time; - u64 cur_wall_time; - u64 busy_time; - - cur_wall_time = jiffies64_to_cputime64(get_jiffies_64()); +/* For cases where we have single governor instance for system */ +static struct cpufreq_interactive_tunables *common_tunables; - busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE]; +static struct attribute_group *get_sysfs_attr(void); - idle_time = cur_wall_time - busy_time; - if (wall) - *wall = jiffies_to_usecs(cur_wall_time); - - return jiffies_to_usecs(idle_time); -} - -static inline cputime64_t get_cpu_idle_time(unsigned int cpu, - cputime64_t *wall) +static void cpufreq_interactive_timer_resched( + struct cpufreq_interactive_cpuinfo *pcpu) { - u64 idle_time = get_cpu_idle_time_us(cpu, wall); + struct cpufreq_interactive_tunables *tunables = + pcpu->policy->governor_data; + unsigned long expires; + unsigned long flags; - if (idle_time == -1ULL) - idle_time = get_cpu_idle_time_jiffy(cpu, wall); - else if (!io_is_busy) - idle_time += get_cpu_iowait_time_us(cpu, wall); + spin_lock_irqsave(&pcpu->load_lock, flags); + pcpu->time_in_idle = + get_cpu_idle_time(smp_processor_id(), + &pcpu->time_in_idle_timestamp, + tunables->io_is_busy); + pcpu->cputime_speedadj = 0; + pcpu->cputime_speedadj_timestamp = pcpu->time_in_idle_timestamp; + expires = jiffies + usecs_to_jiffies(tunables->timer_rate); + mod_timer_pinned(&pcpu->cpu_timer, expires); - return idle_time; + if (tunables->timer_slack_val >= 0 && + pcpu->target_freq > pcpu->policy->min) { + expires += usecs_to_jiffies(tunables->timer_slack_val); + mod_timer_pinned(&pcpu->cpu_slack_timer, expires); + } + + spin_unlock_irqrestore(&pcpu->load_lock, flags); } -static void cpufreq_interactive_timer_resched( - struct cpufreq_interactive_cpuinfo *pcpu) +/* The caller shall take enable_sem write semaphore to avoid any timer race. + * The cpu_timer and cpu_slack_timer must be deactivated when calling this + * function. + */ +static void cpufreq_interactive_timer_start( + struct cpufreq_interactive_tunables *tunables, int cpu) { - unsigned long expires = jiffies + usecs_to_jiffies(timer_rate); + struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, cpu); + unsigned long expires = jiffies + + usecs_to_jiffies(tunables->timer_rate); unsigned long flags; - mod_timer_pinned(&pcpu->cpu_timer, expires); - if (timer_slack_val >= 0 && pcpu->target_freq > pcpu->policy->min) { - expires += usecs_to_jiffies(timer_slack_val); - mod_timer_pinned(&pcpu->cpu_slack_timer, expires); + pcpu->cpu_timer.expires = expires; + add_timer_on(&pcpu->cpu_timer, cpu); + if (tunables->timer_slack_val >= 0 && + pcpu->target_freq > pcpu->policy->min) { + expires += usecs_to_jiffies(tunables->timer_slack_val); + pcpu->cpu_slack_timer.expires = expires; + add_timer_on(&pcpu->cpu_slack_timer, cpu); } spin_lock_irqsave(&pcpu->load_lock, flags); pcpu->time_in_idle = - get_cpu_idle_time(smp_processor_id(), - &pcpu->time_in_idle_timestamp); + get_cpu_idle_time(cpu, &pcpu->time_in_idle_timestamp, + tunables->io_is_busy); pcpu->cputime_speedadj = 0; pcpu->cputime_speedadj_timestamp = pcpu->time_in_idle_timestamp; spin_unlock_irqrestore(&pcpu->load_lock, flags); } -static unsigned int freq_to_above_hispeed_delay(unsigned int freq) +static unsigned int freq_to_above_hispeed_delay( + struct cpufreq_interactive_tunables *tunables, + unsigned int freq) { int i; unsigned int ret; unsigned long flags; - spin_lock_irqsave(&above_hispeed_delay_lock, flags); + spin_lock_irqsave(&tunables->above_hispeed_delay_lock, flags); - for (i = 0; i < nabove_hispeed_delay - 1 && - freq >= above_hispeed_delay[i+1]; i += 2) + for (i = 0; i < tunables->nabove_hispeed_delay - 1 && + freq >= tunables->above_hispeed_delay[i+1]; i += 2) ; - ret = above_hispeed_delay[i]; - spin_unlock_irqrestore(&above_hispeed_delay_lock, flags); + ret = tunables->above_hispeed_delay[i]; + spin_unlock_irqrestore(&tunables->above_hispeed_delay_lock, flags); return ret; } -static unsigned int freq_to_targetload(unsigned int freq) +static unsigned int freq_to_targetload( + struct cpufreq_interactive_tunables *tunables, unsigned int freq) { int i; unsigned int ret; unsigned long flags; - spin_lock_irqsave(&target_loads_lock, flags); + spin_lock_irqsave(&tunables->target_loads_lock, flags); - for (i = 0; i < ntarget_loads - 1 && freq >= target_loads[i+1]; i += 2) + for (i = 0; i < tunables->ntarget_loads - 1 && + freq >= tunables->target_loads[i+1]; i += 2) ; - ret = target_loads[i]; - spin_unlock_irqrestore(&target_loads_lock, flags); + ret = tunables->target_loads[i]; + spin_unlock_irqrestore(&tunables->target_loads_lock, flags); return ret; } @@ -225,9 +231,8 @@ static unsigned int freq_to_targetload(unsigned int freq) * choose_freq() will find the minimum frequency that does not exceed its * target load given the current load. */ - -static unsigned int choose_freq( - struct cpufreq_interactive_cpuinfo *pcpu, unsigned int loadadjfreq) +static unsigned int choose_freq(struct cpufreq_interactive_cpuinfo *pcpu, + unsigned int loadadjfreq) { unsigned int freq = pcpu->policy->cur; unsigned int prevfreq, freqmin, freqmax; @@ -239,16 +244,17 @@ static unsigned int choose_freq( do { prevfreq = freq; - tl = freq_to_targetload(freq); + tl = freq_to_targetload(pcpu->policy->governor_data, freq); /* * Find the lowest frequency where the computed load is less * than or equal to the target load. */ - cpufreq_frequency_table_target( - pcpu->policy, pcpu->freq_table, loadadjfreq / tl, - CPUFREQ_RELATION_L, &index); + if (cpufreq_frequency_table_target( + pcpu->policy, pcpu->freq_table, loadadjfreq / tl, + CPUFREQ_RELATION_L, &index)) + break; freq = pcpu->freq_table[index].frequency; if (freq > prevfreq) { @@ -260,10 +266,11 @@ static unsigned int choose_freq( * Find the highest frequency that is less * than freqmax. */ - cpufreq_frequency_table_target( - pcpu->policy, pcpu->freq_table, - freqmax - 1, CPUFREQ_RELATION_H, - &index); + if (cpufreq_frequency_table_target( + pcpu->policy, pcpu->freq_table, + freqmax - 1, CPUFREQ_RELATION_H, + &index)) + break; freq = pcpu->freq_table[index].frequency; if (freq == freqmin) { @@ -286,10 +293,11 @@ static unsigned int choose_freq( * Find the lowest frequency that is higher * than freqmin. */ - cpufreq_frequency_table_target( - pcpu->policy, pcpu->freq_table, - freqmin + 1, CPUFREQ_RELATION_L, - &index); + if (cpufreq_frequency_table_target( + pcpu->policy, pcpu->freq_table, + freqmin + 1, CPUFREQ_RELATION_L, + &index)) + break; freq = pcpu->freq_table[index].frequency; /* @@ -311,16 +319,23 @@ static unsigned int choose_freq( static u64 update_load(int cpu) { struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, cpu); + struct cpufreq_interactive_tunables *tunables = + pcpu->policy->governor_data; u64 now; u64 now_idle; unsigned int delta_idle; unsigned int delta_time; u64 active_time; - now_idle = get_cpu_idle_time(cpu, &now); + now_idle = get_cpu_idle_time(cpu, &now, tunables->io_is_busy); delta_idle = (unsigned int)(now_idle - pcpu->time_in_idle); delta_time = (unsigned int)(now - pcpu->time_in_idle_timestamp); - active_time = delta_time - delta_idle; + + if (delta_time <= delta_idle) + active_time = 0; + else + active_time = delta_time - delta_idle; + pcpu->cputime_speedadj += active_time * pcpu->policy->cur; pcpu->time_in_idle = now_idle; @@ -336,11 +351,13 @@ static void cpufreq_interactive_timer(unsigned long data) int cpu_load; struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, data); + struct cpufreq_interactive_tunables *tunables = + pcpu->policy->governor_data; unsigned int new_freq; unsigned int loadadjfreq; unsigned int index; unsigned long flags; - bool boosted; + u64 max_fvtime; if (!down_read_trylock(&pcpu->enable_sem)) return; @@ -356,41 +373,50 @@ static void cpufreq_interactive_timer(unsigned long data) if (WARN_ON_ONCE(!delta_time)) goto rearm; + spin_lock_irqsave(&pcpu->target_freq_lock, flags); do_div(cputime_speedadj, delta_time); loadadjfreq = (unsigned int)cputime_speedadj * 100; - cpu_load = loadadjfreq / pcpu->target_freq; - boosted = boost_val || now < boostpulse_endtime; + cpu_load = loadadjfreq / pcpu->policy->cur; + tunables->boosted = tunables->boost_val || now < tunables->boostpulse_endtime; - if (cpu_load >= go_hispeed_load || boosted) { - if (pcpu->target_freq < hispeed_freq) { - new_freq = hispeed_freq; + if (cpu_load >= tunables->go_hispeed_load || tunables->boosted) { + if (pcpu->policy->cur < tunables->hispeed_freq) { + new_freq = tunables->hispeed_freq; } else { new_freq = choose_freq(pcpu, loadadjfreq); - if (new_freq < hispeed_freq) - new_freq = hispeed_freq; + if (new_freq < tunables->hispeed_freq) + new_freq = tunables->hispeed_freq; } } else { new_freq = choose_freq(pcpu, loadadjfreq); + if (new_freq > tunables->hispeed_freq && + pcpu->policy->cur < tunables->hispeed_freq) + new_freq = tunables->hispeed_freq; } - - if (pcpu->target_freq >= hispeed_freq && - new_freq > pcpu->target_freq && - now - pcpu->hispeed_validate_time < - freq_to_above_hispeed_delay(pcpu->target_freq)) { +#ifdef CONFIG_ARCH_ROCKCHIP + if ((now < tunables->touchboostpulse_endtime) && + (new_freq < tunables->touchboost_freq)) { + new_freq = tunables->touchboost_freq; + } +#endif + if (pcpu->policy->cur >= tunables->hispeed_freq && + new_freq > pcpu->policy->cur && + now - pcpu->pol_hispeed_val_time < + freq_to_above_hispeed_delay(tunables, pcpu->policy->cur)) { trace_cpufreq_interactive_notyet( data, cpu_load, pcpu->target_freq, pcpu->policy->cur, new_freq); + spin_unlock_irqrestore(&pcpu->target_freq_lock, flags); goto rearm; } - pcpu->hispeed_validate_time = now; + pcpu->loc_hispeed_val_time = now; if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table, new_freq, CPUFREQ_RELATION_L, &index)) { - pr_warn_once("timer %d: cpufreq_frequency_table_target error\n", - (int) data); + spin_unlock_irqrestore(&pcpu->target_freq_lock, flags); goto rearm; } @@ -400,11 +426,14 @@ static void cpufreq_interactive_timer(unsigned long data) * Do not scale below floor_freq unless we have been at or above the * floor frequency for the minimum sample time since last validated. */ - if (new_freq < pcpu->floor_freq) { - if (now - pcpu->floor_validate_time < min_sample_time) { + max_fvtime = max(pcpu->pol_floor_val_time, pcpu->loc_floor_val_time); + if (new_freq < pcpu->floor_freq && + pcpu->target_freq >= pcpu->policy->cur) { + if (now - max_fvtime < tunables->min_sample_time) { trace_cpufreq_interactive_notyet( data, cpu_load, pcpu->target_freq, pcpu->policy->cur, new_freq); + spin_unlock_irqrestore(&pcpu->target_freq_lock, flags); goto rearm; } } @@ -417,35 +446,32 @@ static void cpufreq_interactive_timer(unsigned long data) * (or the indefinite boost is turned off). */ - if (!boosted || new_freq > hispeed_freq) { + if (!tunables->boosted || new_freq > tunables->hispeed_freq) { pcpu->floor_freq = new_freq; - pcpu->floor_validate_time = now; + if (pcpu->target_freq >= pcpu->policy->cur || + new_freq >= pcpu->policy->cur) + pcpu->loc_floor_val_time = now; } - if (pcpu->target_freq == new_freq) { + if (pcpu->target_freq == new_freq && + pcpu->target_freq <= pcpu->policy->cur) { trace_cpufreq_interactive_already( data, cpu_load, pcpu->target_freq, pcpu->policy->cur, new_freq); - goto rearm_if_notmax; + spin_unlock_irqrestore(&pcpu->target_freq_lock, flags); + goto rearm; } trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq, pcpu->policy->cur, new_freq); pcpu->target_freq = new_freq; + spin_unlock_irqrestore(&pcpu->target_freq_lock, flags); spin_lock_irqsave(&speedchange_cpumask_lock, flags); cpumask_set_cpu(data, &speedchange_cpumask); spin_unlock_irqrestore(&speedchange_cpumask_lock, flags); wake_up_process(speedchange_task); -rearm_if_notmax: - /* - * Already set max speed and don't see a need to change that, - * wait until next idle to re-evaluate, don't need timer. - */ - if (pcpu->target_freq == pcpu->policy->max) - goto exit; - rearm: if (!timer_pending(&pcpu->cpu_timer)) cpufreq_interactive_timer_resched(pcpu); @@ -455,37 +481,6 @@ exit: return; } -static void cpufreq_interactive_idle_start(void) -{ - struct cpufreq_interactive_cpuinfo *pcpu = - &per_cpu(cpuinfo, smp_processor_id()); - int pending; - - if (!down_read_trylock(&pcpu->enable_sem)) - return; - if (!pcpu->governor_enabled) { - up_read(&pcpu->enable_sem); - return; - } - - pending = timer_pending(&pcpu->cpu_timer); - - if (pcpu->target_freq != pcpu->policy->min) { - /* - * Entering idle while not at lowest speed. On some - * platforms this can hold the other CPU(s) at that speed - * even though the CPU is idle. Set a timer to re-evaluate - * speed so this idle CPU doesn't hold the other CPUs above - * min indefinitely. This should probably be a quirk of - * the CPUFreq driver. - */ - if (!pending) - cpufreq_interactive_timer_resched(pcpu); - } - - up_read(&pcpu->enable_sem); -} - static void cpufreq_interactive_idle_end(void) { struct cpufreq_interactive_cpuinfo *pcpu = @@ -510,6 +505,58 @@ static void cpufreq_interactive_idle_end(void) up_read(&pcpu->enable_sem); } +static void cpufreq_interactive_get_policy_info(struct cpufreq_policy *policy, + unsigned int *pmax_freq, + u64 *phvt, u64 *pfvt) +{ + struct cpufreq_interactive_cpuinfo *pcpu; + unsigned int max_freq = 0; + u64 hvt = ~0ULL, fvt = 0; + unsigned int i; + + for_each_cpu(i, policy->cpus) { + pcpu = &per_cpu(cpuinfo, i); + + fvt = max(fvt, pcpu->loc_floor_val_time); + if (pcpu->target_freq > max_freq) { + max_freq = pcpu->target_freq; + hvt = pcpu->loc_hispeed_val_time; + } else if (pcpu->target_freq == max_freq) { + hvt = min(hvt, pcpu->loc_hispeed_val_time); + } + } + + *pmax_freq = max_freq; + *phvt = hvt; + *pfvt = fvt; +} + +static void cpufreq_interactive_adjust_cpu(unsigned int cpu, + struct cpufreq_policy *policy) +{ + struct cpufreq_interactive_cpuinfo *pcpu; + u64 hvt, fvt; + unsigned int max_freq; + int i; + + cpufreq_interactive_get_policy_info(policy, &max_freq, &hvt, &fvt); + + for_each_cpu(i, policy->cpus) { + pcpu = &per_cpu(cpuinfo, i); + pcpu->pol_floor_val_time = fvt; + } + + if (max_freq != policy->cur) { + __cpufreq_driver_target(policy, max_freq, CPUFREQ_RELATION_H); + for_each_cpu(i, policy->cpus) { + pcpu = &per_cpu(cpuinfo, i); + pcpu->pol_hispeed_val_time = hvt; + } + } + + trace_cpufreq_interactive_setspeed(cpu, max_freq, policy->cur); +} + static int cpufreq_interactive_speedchange_task(void *data) { unsigned int cpu; @@ -538,70 +585,65 @@ static int cpufreq_interactive_speedchange_task(void *data) spin_unlock_irqrestore(&speedchange_cpumask_lock, flags); for_each_cpu(cpu, &tmp_mask) { - unsigned int j; - unsigned int max_freq = 0; - pcpu = &per_cpu(cpuinfo, cpu); - if (!down_read_trylock(&pcpu->enable_sem)) - continue; - if (!pcpu->governor_enabled) { - up_read(&pcpu->enable_sem); - continue; - } - for_each_cpu(j, pcpu->policy->cpus) { - struct cpufreq_interactive_cpuinfo *pjcpu = - &per_cpu(cpuinfo, j); + down_write(&pcpu->policy->rwsem); - if (pjcpu->target_freq > max_freq) - max_freq = pjcpu->target_freq; + if (likely(down_read_trylock(&pcpu->enable_sem))) { + if (likely(pcpu->governor_enabled)) + cpufreq_interactive_adjust_cpu(cpu, + pcpu->policy); + up_read(&pcpu->enable_sem); } - if (max_freq != pcpu->policy->cur) - __cpufreq_driver_target(pcpu->policy, - max_freq, - CPUFREQ_RELATION_H); - trace_cpufreq_interactive_setspeed(cpu, - pcpu->target_freq, - pcpu->policy->cur); - - up_read(&pcpu->enable_sem); + up_write(&pcpu->policy->rwsem); } } return 0; } -static void cpufreq_interactive_boost(void) +static void cpufreq_interactive_boost(struct cpufreq_interactive_tunables *tunables) { int i; int anyboost = 0; - unsigned long flags; + unsigned long flags[2]; struct cpufreq_interactive_cpuinfo *pcpu; - spin_lock_irqsave(&speedchange_cpumask_lock, flags); + tunables->boosted = true; + + spin_lock_irqsave(&speedchange_cpumask_lock, flags[0]); for_each_online_cpu(i) { pcpu = &per_cpu(cpuinfo, i); - if (pcpu->target_freq < hispeed_freq) { - pcpu->target_freq = hispeed_freq; + if (!down_read_trylock(&pcpu->enable_sem)) + continue; + + if (!pcpu->governor_enabled) { + up_read(&pcpu->enable_sem); + continue; + } + + if (tunables != pcpu->policy->governor_data) { + up_read(&pcpu->enable_sem); + continue; + } + + spin_lock_irqsave(&pcpu->target_freq_lock, flags[1]); + if (pcpu->target_freq < tunables->hispeed_freq) { + pcpu->target_freq = tunables->hispeed_freq; cpumask_set_cpu(i, &speedchange_cpumask); - pcpu->hispeed_validate_time = + pcpu->pol_hispeed_val_time = ktime_to_us(ktime_get()); anyboost = 1; } + spin_unlock_irqrestore(&pcpu->target_freq_lock, flags[1]); - /* - * Set floor freq and (re)start timer for when last - * validated. - */ - - pcpu->floor_freq = hispeed_freq; - pcpu->floor_validate_time = ktime_to_us(ktime_get()); + up_read(&pcpu->enable_sem); } - spin_unlock_irqrestore(&speedchange_cpumask_lock, flags); + spin_unlock_irqrestore(&speedchange_cpumask_lock, flags[0]); if (anyboost) wake_up_process(speedchange_task); @@ -627,9 +669,19 @@ static int cpufreq_interactive_notifier( for_each_cpu(cpu, pcpu->policy->cpus) { struct cpufreq_interactive_cpuinfo *pjcpu = &per_cpu(cpuinfo, cpu); + if (cpu != freq->cpu) { + if (!down_read_trylock(&pjcpu->enable_sem)) + continue; + if (!pjcpu->governor_enabled) { + up_read(&pjcpu->enable_sem); + continue; + } + } spin_lock_irqsave(&pjcpu->load_lock, flags); update_load(cpu); spin_unlock_irqrestore(&pjcpu->load_lock, flags); + if (cpu != freq->cpu) + up_read(&pjcpu->enable_sem); } up_read(&pcpu->enable_sem); @@ -687,26 +739,27 @@ err: } static ssize_t show_target_loads( - struct kobject *kobj, struct attribute *attr, char *buf) + struct cpufreq_interactive_tunables *tunables, + char *buf) { int i; ssize_t ret = 0; unsigned long flags; - spin_lock_irqsave(&target_loads_lock, flags); + spin_lock_irqsave(&tunables->target_loads_lock, flags); - for (i = 0; i < ntarget_loads; i++) - ret += sprintf(buf + ret, "%u%s", target_loads[i], + for (i = 0; i < tunables->ntarget_loads; i++) + ret += sprintf(buf + ret, "%u%s", tunables->target_loads[i], i & 0x1 ? ":" : " "); - ret += sprintf(buf + ret, "\n"); - spin_unlock_irqrestore(&target_loads_lock, flags); + sprintf(buf + ret - 1, "\n"); + spin_unlock_irqrestore(&tunables->target_loads_lock, flags); return ret; } static ssize_t store_target_loads( - struct kobject *kobj, struct attribute *attr, const char *buf, - size_t count) + struct cpufreq_interactive_tunables *tunables, + const char *buf, size_t count) { int ntokens; unsigned int *new_target_loads = NULL; @@ -716,40 +769,37 @@ static ssize_t store_target_loads( if (IS_ERR(new_target_loads)) return PTR_RET(new_target_loads); - spin_lock_irqsave(&target_loads_lock, flags); - if (target_loads != default_target_loads) - kfree(target_loads); - target_loads = new_target_loads; - ntarget_loads = ntokens; - spin_unlock_irqrestore(&target_loads_lock, flags); + spin_lock_irqsave(&tunables->target_loads_lock, flags); + if (tunables->target_loads != default_target_loads) + kfree(tunables->target_loads); + tunables->target_loads = new_target_loads; + tunables->ntarget_loads = ntokens; + spin_unlock_irqrestore(&tunables->target_loads_lock, flags); return count; } -static struct global_attr target_loads_attr = - __ATTR(target_loads, S_IRUGO | S_IWUSR, - show_target_loads, store_target_loads); - static ssize_t show_above_hispeed_delay( - struct kobject *kobj, struct attribute *attr, char *buf) + struct cpufreq_interactive_tunables *tunables, char *buf) { int i; ssize_t ret = 0; unsigned long flags; - spin_lock_irqsave(&above_hispeed_delay_lock, flags); + spin_lock_irqsave(&tunables->above_hispeed_delay_lock, flags); - for (i = 0; i < nabove_hispeed_delay; i++) - ret += sprintf(buf + ret, "%u%s", above_hispeed_delay[i], + for (i = 0; i < tunables->nabove_hispeed_delay; i++) + ret += sprintf(buf + ret, "%u%s", + tunables->above_hispeed_delay[i], i & 0x1 ? ":" : " "); - ret += sprintf(buf + ret, "\n"); - spin_unlock_irqrestore(&above_hispeed_delay_lock, flags); + sprintf(buf + ret - 1, "\n"); + spin_unlock_irqrestore(&tunables->above_hispeed_delay_lock, flags); return ret; } static ssize_t store_above_hispeed_delay( - struct kobject *kobj, struct attribute *attr, const char *buf, - size_t count) + struct cpufreq_interactive_tunables *tunables, + const char *buf, size_t count) { int ntokens; unsigned int *new_above_hispeed_delay = NULL; @@ -759,119 +809,106 @@ static ssize_t store_above_hispeed_delay( if (IS_ERR(new_above_hispeed_delay)) return PTR_RET(new_above_hispeed_delay); - spin_lock_irqsave(&above_hispeed_delay_lock, flags); - if (above_hispeed_delay != default_above_hispeed_delay) - kfree(above_hispeed_delay); - above_hispeed_delay = new_above_hispeed_delay; - nabove_hispeed_delay = ntokens; - spin_unlock_irqrestore(&above_hispeed_delay_lock, flags); + spin_lock_irqsave(&tunables->above_hispeed_delay_lock, flags); + if (tunables->above_hispeed_delay != default_above_hispeed_delay) + kfree(tunables->above_hispeed_delay); + tunables->above_hispeed_delay = new_above_hispeed_delay; + tunables->nabove_hispeed_delay = ntokens; + spin_unlock_irqrestore(&tunables->above_hispeed_delay_lock, flags); return count; } -static struct global_attr above_hispeed_delay_attr = - __ATTR(above_hispeed_delay, S_IRUGO | S_IWUSR, - show_above_hispeed_delay, store_above_hispeed_delay); - -static ssize_t show_hispeed_freq(struct kobject *kobj, - struct attribute *attr, char *buf) +static ssize_t show_hispeed_freq(struct cpufreq_interactive_tunables *tunables, + char *buf) { - return sprintf(buf, "%u\n", hispeed_freq); + return sprintf(buf, "%u\n", tunables->hispeed_freq); } -static ssize_t store_hispeed_freq(struct kobject *kobj, - struct attribute *attr, const char *buf, - size_t count) +static ssize_t store_hispeed_freq(struct cpufreq_interactive_tunables *tunables, + const char *buf, size_t count) { int ret; long unsigned int val; - ret = strict_strtoul(buf, 0, &val); + ret = kstrtoul(buf, 0, &val); if (ret < 0) return ret; - hispeed_freq = val; + tunables->hispeed_freq = val; return count; } -static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644, - show_hispeed_freq, store_hispeed_freq); - - -static ssize_t show_go_hispeed_load(struct kobject *kobj, - struct attribute *attr, char *buf) +static ssize_t show_go_hispeed_load(struct cpufreq_interactive_tunables + *tunables, char *buf) { - return sprintf(buf, "%lu\n", go_hispeed_load); + return sprintf(buf, "%lu\n", tunables->go_hispeed_load); } -static ssize_t store_go_hispeed_load(struct kobject *kobj, - struct attribute *attr, const char *buf, size_t count) +static ssize_t store_go_hispeed_load(struct cpufreq_interactive_tunables + *tunables, const char *buf, size_t count) { int ret; unsigned long val; - ret = strict_strtoul(buf, 0, &val); + ret = kstrtoul(buf, 0, &val); if (ret < 0) return ret; - go_hispeed_load = val; + tunables->go_hispeed_load = val; return count; } -static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644, - show_go_hispeed_load, store_go_hispeed_load); - -static ssize_t show_min_sample_time(struct kobject *kobj, - struct attribute *attr, char *buf) +static ssize_t show_min_sample_time(struct cpufreq_interactive_tunables + *tunables, char *buf) { - return sprintf(buf, "%lu\n", min_sample_time); + return sprintf(buf, "%lu\n", tunables->min_sample_time); } -static ssize_t store_min_sample_time(struct kobject *kobj, - struct attribute *attr, const char *buf, size_t count) +static ssize_t store_min_sample_time(struct cpufreq_interactive_tunables + *tunables, const char *buf, size_t count) { int ret; unsigned long val; - ret = strict_strtoul(buf, 0, &val); + ret = kstrtoul(buf, 0, &val); if (ret < 0) return ret; - min_sample_time = val; + tunables->min_sample_time = val; return count; } -static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644, - show_min_sample_time, store_min_sample_time); - -static ssize_t show_timer_rate(struct kobject *kobj, - struct attribute *attr, char *buf) +static ssize_t show_timer_rate(struct cpufreq_interactive_tunables *tunables, + char *buf) { - return sprintf(buf, "%lu\n", timer_rate); + return sprintf(buf, "%lu\n", tunables->timer_rate); } -static ssize_t store_timer_rate(struct kobject *kobj, - struct attribute *attr, const char *buf, size_t count) +static ssize_t store_timer_rate(struct cpufreq_interactive_tunables *tunables, + const char *buf, size_t count) { int ret; - unsigned long val; + unsigned long val, val_round; - ret = strict_strtoul(buf, 0, &val); + ret = kstrtoul(buf, 0, &val); if (ret < 0) return ret; - timer_rate = val; + + val_round = jiffies_to_usecs(usecs_to_jiffies(val)); + if (val != val_round) + pr_warn("timer_rate not aligned to jiffy. Rounded up to %lu\n", + val_round); + + tunables->timer_rate = val_round; return count; } -static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644, - show_timer_rate, store_timer_rate); - -static ssize_t show_timer_slack( - struct kobject *kobj, struct attribute *attr, char *buf) +static ssize_t show_timer_slack(struct cpufreq_interactive_tunables *tunables, + char *buf) { - return sprintf(buf, "%d\n", timer_slack_val); + return sprintf(buf, "%d\n", tunables->timer_slack_val); } -static ssize_t store_timer_slack( - struct kobject *kobj, struct attribute *attr, const char *buf, - size_t count) +static ssize_t store_timer_slack(struct cpufreq_interactive_tunables *tunables, + const char *buf, size_t count) { int ret; unsigned long val; @@ -880,19 +917,17 @@ static ssize_t store_timer_slack( if (ret < 0) return ret; - timer_slack_val = val; + tunables->timer_slack_val = val; return count; } -define_one_global_rw(timer_slack); - -static ssize_t show_boost(struct kobject *kobj, struct attribute *attr, +static ssize_t show_boost(struct cpufreq_interactive_tunables *tunables, char *buf) { - return sprintf(buf, "%d\n", boost_val); + return sprintf(buf, "%d\n", tunables->boost_val); } -static ssize_t store_boost(struct kobject *kobj, struct attribute *attr, +static ssize_t store_boost(struct cpufreq_interactive_tunables *tunables, const char *buf, size_t count) { int ret; @@ -902,21 +937,21 @@ static ssize_t store_boost(struct kobject *kobj, struct attribute *attr, if (ret < 0) return ret; - boost_val = val; + tunables->boost_val = val; - if (boost_val) { + if (tunables->boost_val) { trace_cpufreq_interactive_boost("on"); - cpufreq_interactive_boost(); + if (!tunables->boosted) + cpufreq_interactive_boost(tunables); } else { + tunables->boostpulse_endtime = ktime_to_us(ktime_get()); trace_cpufreq_interactive_unboost("off"); } return count; } -define_one_global_rw(boost); - -static ssize_t store_boostpulse(struct kobject *kobj, struct attribute *attr, +static ssize_t store_boostpulse(struct cpufreq_interactive_tunables *tunables, const char *buf, size_t count) { int ret; @@ -926,24 +961,22 @@ static ssize_t store_boostpulse(struct kobject *kobj, struct attribute *attr, if (ret < 0) return ret; - boostpulse_endtime = ktime_to_us(ktime_get()) + boostpulse_duration_val; + tunables->boostpulse_endtime = ktime_to_us(ktime_get()) + + tunables->boostpulse_duration_val; trace_cpufreq_interactive_boost("pulse"); - cpufreq_interactive_boost(); + if (!tunables->boosted) + cpufreq_interactive_boost(tunables); return count; } -static struct global_attr boostpulse = - __ATTR(boostpulse, 0200, NULL, store_boostpulse); - -static ssize_t show_boostpulse_duration( - struct kobject *kobj, struct attribute *attr, char *buf) +static ssize_t show_boostpulse_duration(struct cpufreq_interactive_tunables + *tunables, char *buf) { - return sprintf(buf, "%d\n", boostpulse_duration_val); + return sprintf(buf, "%d\n", tunables->boostpulse_duration_val); } -static ssize_t store_boostpulse_duration( - struct kobject *kobj, struct attribute *attr, const char *buf, - size_t count) +static ssize_t store_boostpulse_duration(struct cpufreq_interactive_tunables + *tunables, const char *buf, size_t count) { int ret; unsigned long val; @@ -952,20 +985,18 @@ static ssize_t store_boostpulse_duration( if (ret < 0) return ret; - boostpulse_duration_val = val; + tunables->boostpulse_duration_val = val; return count; } -define_one_global_rw(boostpulse_duration); - -static ssize_t show_io_is_busy(struct kobject *kobj, - struct attribute *attr, char *buf) +static ssize_t show_io_is_busy(struct cpufreq_interactive_tunables *tunables, + char *buf) { - return sprintf(buf, "%u\n", io_is_busy); + return sprintf(buf, "%u\n", tunables->io_is_busy); } -static ssize_t store_io_is_busy(struct kobject *kobj, - struct attribute *attr, const char *buf, size_t count) +static ssize_t store_io_is_busy(struct cpufreq_interactive_tunables *tunables, + const char *buf, size_t count) { int ret; unsigned long val; @@ -973,45 +1004,143 @@ static ssize_t store_io_is_busy(struct kobject *kobj, ret = kstrtoul(buf, 0, &val); if (ret < 0) return ret; - io_is_busy = val; + tunables->io_is_busy = val; return count; } -static struct global_attr io_is_busy_attr = __ATTR(io_is_busy, 0644, - show_io_is_busy, store_io_is_busy); - -static struct attribute *interactive_attributes[] = { - &target_loads_attr.attr, - &above_hispeed_delay_attr.attr, - &hispeed_freq_attr.attr, - &go_hispeed_load_attr.attr, - &min_sample_time_attr.attr, - &timer_rate_attr.attr, - &timer_slack.attr, - &boost.attr, - &boostpulse.attr, - &boostpulse_duration.attr, - &io_is_busy_attr.attr, +/* + * Create show/store routines + * - sys: One governor instance for complete SYSTEM + * - pol: One governor instance per struct cpufreq_policy + */ +#define show_gov_pol_sys(file_name) \ +static ssize_t show_##file_name##_gov_sys \ +(struct kobject *kobj, struct attribute *attr, char *buf) \ +{ \ + return show_##file_name(common_tunables, buf); \ +} \ + \ +static ssize_t show_##file_name##_gov_pol \ +(struct cpufreq_policy *policy, char *buf) \ +{ \ + return show_##file_name(policy->governor_data, buf); \ +} + +#define store_gov_pol_sys(file_name) \ +static ssize_t store_##file_name##_gov_sys \ +(struct kobject *kobj, struct attribute *attr, const char *buf, \ + size_t count) \ +{ \ + return store_##file_name(common_tunables, buf, count); \ +} \ + \ +static ssize_t store_##file_name##_gov_pol \ +(struct cpufreq_policy *policy, const char *buf, size_t count) \ +{ \ + return store_##file_name(policy->governor_data, buf, count); \ +} + +#define show_store_gov_pol_sys(file_name) \ +show_gov_pol_sys(file_name); \ +store_gov_pol_sys(file_name) + +show_store_gov_pol_sys(target_loads); +show_store_gov_pol_sys(above_hispeed_delay); +show_store_gov_pol_sys(hispeed_freq); +show_store_gov_pol_sys(go_hispeed_load); +show_store_gov_pol_sys(min_sample_time); +show_store_gov_pol_sys(timer_rate); +show_store_gov_pol_sys(timer_slack); +show_store_gov_pol_sys(boost); +store_gov_pol_sys(boostpulse); +show_store_gov_pol_sys(boostpulse_duration); +show_store_gov_pol_sys(io_is_busy); + +#define gov_sys_attr_rw(_name) \ +static struct global_attr _name##_gov_sys = \ +__ATTR(_name, 0644, show_##_name##_gov_sys, store_##_name##_gov_sys) + +#define gov_pol_attr_rw(_name) \ +static struct freq_attr _name##_gov_pol = \ +__ATTR(_name, 0644, show_##_name##_gov_pol, store_##_name##_gov_pol) + +#define gov_sys_pol_attr_rw(_name) \ + gov_sys_attr_rw(_name); \ + gov_pol_attr_rw(_name) + +gov_sys_pol_attr_rw(target_loads); +gov_sys_pol_attr_rw(above_hispeed_delay); +gov_sys_pol_attr_rw(hispeed_freq); +gov_sys_pol_attr_rw(go_hispeed_load); +gov_sys_pol_attr_rw(min_sample_time); +gov_sys_pol_attr_rw(timer_rate); +gov_sys_pol_attr_rw(timer_slack); +gov_sys_pol_attr_rw(boost); +gov_sys_pol_attr_rw(boostpulse_duration); +gov_sys_pol_attr_rw(io_is_busy); + +static struct global_attr boostpulse_gov_sys = + __ATTR(boostpulse, 0200, NULL, store_boostpulse_gov_sys); + +static struct freq_attr boostpulse_gov_pol = + __ATTR(boostpulse, 0200, NULL, store_boostpulse_gov_pol); + +/* One Governor instance for entire system */ +static struct attribute *interactive_attributes_gov_sys[] = { + &target_loads_gov_sys.attr, + &above_hispeed_delay_gov_sys.attr, + &hispeed_freq_gov_sys.attr, + &go_hispeed_load_gov_sys.attr, + &min_sample_time_gov_sys.attr, + &timer_rate_gov_sys.attr, + &timer_slack_gov_sys.attr, + &boost_gov_sys.attr, + &boostpulse_gov_sys.attr, + &boostpulse_duration_gov_sys.attr, + &io_is_busy_gov_sys.attr, NULL, }; -static struct attribute_group interactive_attr_group = { - .attrs = interactive_attributes, +static struct attribute_group interactive_attr_group_gov_sys = { + .attrs = interactive_attributes_gov_sys, .name = "interactive", }; +/* Per policy governor instance */ +static struct attribute *interactive_attributes_gov_pol[] = { + &target_loads_gov_pol.attr, + &above_hispeed_delay_gov_pol.attr, + &hispeed_freq_gov_pol.attr, + &go_hispeed_load_gov_pol.attr, + &min_sample_time_gov_pol.attr, + &timer_rate_gov_pol.attr, + &timer_slack_gov_pol.attr, + &boost_gov_pol.attr, + &boostpulse_gov_pol.attr, + &boostpulse_duration_gov_pol.attr, + &io_is_busy_gov_pol.attr, + NULL, +}; + +static struct attribute_group interactive_attr_group_gov_pol = { + .attrs = interactive_attributes_gov_pol, + .name = "interactive", +}; + +static struct attribute_group *get_sysfs_attr(void) +{ + if (have_governor_per_policy()) + return &interactive_attr_group_gov_pol; + else + return &interactive_attr_group_gov_sys; +} + static int cpufreq_interactive_idle_notifier(struct notifier_block *nb, unsigned long val, void *data) { - switch (val) { - case IDLE_START: - cpufreq_interactive_idle_start(); - break; - case IDLE_END: + if (val == IDLE_END) cpufreq_interactive_idle_end(); - break; - } return 0; } @@ -1020,6 +1149,147 @@ static struct notifier_block cpufreq_interactive_idle_nb = { .notifier_call = cpufreq_interactive_idle_notifier, }; +#ifdef CONFIG_ARCH_ROCKCHIP +static void cpufreq_interactive_input_event(struct input_handle *handle, + unsigned int type, + unsigned int code, + int value) +{ + u64 now, endtime; + int i; + int anyboost = 0; + unsigned long flags[2]; + struct cpufreq_interactive_cpuinfo *pcpu; + struct cpufreq_interactive_tunables *tunables; + + if ((type != EV_ABS) && (type != EV_KEY)) + return; + + trace_cpufreq_interactive_boost("touch"); + spin_lock_irqsave(&speedchange_cpumask_lock, flags[0]); + + now = ktime_to_us(ktime_get()); + for_each_online_cpu(i) { + pcpu = &per_cpu(cpuinfo, i); + if (have_governor_per_policy()) + tunables = pcpu->policy->governor_data; + else + tunables = common_tunables; + if (!tunables) + continue; + + endtime = now + tunables->touchboostpulse_duration_val; + if (endtime < (tunables->touchboostpulse_endtime + + 10 * USEC_PER_MSEC)) + continue; + tunables->touchboostpulse_endtime = endtime; + + spin_lock_irqsave(&pcpu->target_freq_lock, flags[1]); + if (pcpu->target_freq < tunables->touchboost_freq) { + pcpu->target_freq = tunables->touchboost_freq; + cpumask_set_cpu(i, &speedchange_cpumask); + pcpu->loc_hispeed_val_time = + ktime_to_us(ktime_get()); + anyboost = 1; + } + + pcpu->floor_freq = tunables->touchboost_freq; + pcpu->loc_floor_val_time = ktime_to_us(ktime_get()); + + spin_unlock_irqrestore(&pcpu->target_freq_lock, flags[1]); + } + + spin_unlock_irqrestore(&speedchange_cpumask_lock, flags[0]); + + if (anyboost) + wake_up_process(speedchange_task); +} + +static int cpufreq_interactive_input_connect(struct input_handler *handler, + struct input_dev *dev, + const struct input_device_id *id) +{ + struct input_handle *handle; + int error; + + handle = kzalloc(sizeof(*handle), GFP_KERNEL); + if (!handle) + return -ENOMEM; + + handle->dev = dev; + handle->handler = handler; + handle->name = "cpufreq"; + + error = input_register_handle(handle); + if (error) + goto err2; + + error = input_open_device(handle); + if (error) + goto err1; + + return 0; +err1: + input_unregister_handle(handle); +err2: + kfree(handle); + return error; +} + +static void cpufreq_interactive_input_disconnect(struct input_handle *handle) +{ + input_close_device(handle); + input_unregister_handle(handle); + kfree(handle); +} + +static const struct input_device_id cpufreq_interactive_ids[] = { + { + .flags = INPUT_DEVICE_ID_MATCH_EVBIT | + INPUT_DEVICE_ID_MATCH_ABSBIT, + .evbit = { BIT_MASK(EV_ABS) }, + .absbit = { [BIT_WORD(ABS_MT_POSITION_X)] = + BIT_MASK(ABS_MT_POSITION_X) | + BIT_MASK(ABS_MT_POSITION_Y) }, + }, + { + .flags = INPUT_DEVICE_ID_MATCH_KEYBIT | + INPUT_DEVICE_ID_MATCH_ABSBIT, + .keybit = { [BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH) }, + .absbit = { [BIT_WORD(ABS_X)] = + BIT_MASK(ABS_X) | BIT_MASK(ABS_Y) }, + }, + { + .flags = INPUT_DEVICE_ID_MATCH_EVBIT, + .evbit = { BIT_MASK(EV_KEY) }, + }, + { }, +}; + +static struct input_handler cpufreq_interactive_input_handler = { + .event = cpufreq_interactive_input_event, + .connect = cpufreq_interactive_input_connect, + .disconnect = cpufreq_interactive_input_disconnect, + .name = "cpufreq_interactive", + .id_table = cpufreq_interactive_ids, +}; + +static void rockchip_cpufreq_policy_init(struct cpufreq_policy *policy) +{ + struct cpufreq_interactive_tunables *tunables = policy->governor_data; + + tunables->min_sample_time = 40 * USEC_PER_MSEC; + tunables->boostpulse_duration_val = 40 * USEC_PER_MSEC; + if (policy->cpu == 0) { + tunables->hispeed_freq = 1008000; + tunables->touchboostpulse_duration_val = 500 * USEC_PER_MSEC; + tunables->touchboost_freq = 1200000; + } else { + tunables->hispeed_freq = 816000; + } +} +#endif + static int cpufreq_governor_interactive(struct cpufreq_policy *policy, unsigned int event) { @@ -1027,63 +1297,126 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy, unsigned int j; struct cpufreq_interactive_cpuinfo *pcpu; struct cpufreq_frequency_table *freq_table; + struct cpufreq_interactive_tunables *tunables; + unsigned long flags; + + if (have_governor_per_policy()) + tunables = policy->governor_data; + else + tunables = common_tunables; + + WARN_ON(!tunables && (event != CPUFREQ_GOV_POLICY_INIT)); switch (event) { - case CPUFREQ_GOV_START: - if (!cpu_online(policy->cpu)) - return -EINVAL; + case CPUFREQ_GOV_POLICY_INIT: + if (have_governor_per_policy()) { + WARN_ON(tunables); + } else if (tunables) { + tunables->usage_count++; + policy->governor_data = tunables; + return 0; + } + + tunables = kzalloc(sizeof(*tunables), GFP_KERNEL); + if (!tunables) { + pr_err("%s: POLICY_INIT: kzalloc failed\n", __func__); + return -ENOMEM; + } + + tunables->usage_count = 1; + tunables->above_hispeed_delay = default_above_hispeed_delay; + tunables->nabove_hispeed_delay = + ARRAY_SIZE(default_above_hispeed_delay); + tunables->go_hispeed_load = DEFAULT_GO_HISPEED_LOAD; + tunables->target_loads = default_target_loads; + tunables->ntarget_loads = ARRAY_SIZE(default_target_loads); + tunables->min_sample_time = DEFAULT_MIN_SAMPLE_TIME; + tunables->timer_rate = DEFAULT_TIMER_RATE; + tunables->boostpulse_duration_val = DEFAULT_MIN_SAMPLE_TIME; + tunables->timer_slack_val = DEFAULT_TIMER_SLACK; + + spin_lock_init(&tunables->target_loads_lock); + spin_lock_init(&tunables->above_hispeed_delay_lock); + + policy->governor_data = tunables; + if (!have_governor_per_policy()) { + common_tunables = tunables; + } + +#ifdef CONFIG_ARCH_ROCKCHIP + rockchip_cpufreq_policy_init(policy); +#endif + + rc = sysfs_create_group(get_governor_parent_kobj(policy), + get_sysfs_attr()); + if (rc) { + kfree(tunables); + policy->governor_data = NULL; + if (!have_governor_per_policy()) { + common_tunables = NULL; + } + return rc; + } + + if (!policy->governor->initialized) { + idle_notifier_register(&cpufreq_interactive_idle_nb); + cpufreq_register_notifier(&cpufreq_notifier_block, + CPUFREQ_TRANSITION_NOTIFIER); +#ifdef CONFIG_ARCH_ROCKCHIP + rc = input_register_handler(&cpufreq_interactive_input_handler); +#endif + + } + + break; + case CPUFREQ_GOV_POLICY_EXIT: + if (!--tunables->usage_count) { + if (policy->governor->initialized == 1) { +#ifdef CONFIG_ARCH_ROCKCHIP + input_unregister_handler(&cpufreq_interactive_input_handler); +#endif + cpufreq_unregister_notifier(&cpufreq_notifier_block, + CPUFREQ_TRANSITION_NOTIFIER); + idle_notifier_unregister(&cpufreq_interactive_idle_nb); + } + + sysfs_remove_group(get_governor_parent_kobj(policy), + get_sysfs_attr()); + + kfree(tunables); + common_tunables = NULL; + } + + policy->governor_data = NULL; + break; + + case CPUFREQ_GOV_START: mutex_lock(&gov_lock); - freq_table = - cpufreq_frequency_get_table(policy->cpu); - if (!hispeed_freq) - hispeed_freq = policy->max; + freq_table = cpufreq_frequency_get_table(policy->cpu); + if (!tunables->hispeed_freq) + tunables->hispeed_freq = policy->max; for_each_cpu(j, policy->cpus) { - unsigned long expires; - pcpu = &per_cpu(cpuinfo, j); pcpu->policy = policy; pcpu->target_freq = policy->cur; pcpu->freq_table = freq_table; pcpu->floor_freq = pcpu->target_freq; - pcpu->floor_validate_time = + pcpu->pol_floor_val_time = ktime_to_us(ktime_get()); - pcpu->hispeed_validate_time = - pcpu->floor_validate_time; + pcpu->loc_floor_val_time = pcpu->pol_floor_val_time; + pcpu->pol_hispeed_val_time = pcpu->pol_floor_val_time; + pcpu->loc_hispeed_val_time = pcpu->pol_floor_val_time; down_write(&pcpu->enable_sem); - expires = jiffies + usecs_to_jiffies(timer_rate); - pcpu->cpu_timer.expires = expires; - add_timer_on(&pcpu->cpu_timer, j); - if (timer_slack_val >= 0) { - expires += usecs_to_jiffies(timer_slack_val); - pcpu->cpu_slack_timer.expires = expires; - add_timer_on(&pcpu->cpu_slack_timer, j); - } + del_timer_sync(&pcpu->cpu_timer); + del_timer_sync(&pcpu->cpu_slack_timer); + cpufreq_interactive_timer_start(tunables, j); pcpu->governor_enabled = 1; up_write(&pcpu->enable_sem); } - /* - * Do not register the idle hook and create sysfs - * entries if we have already done so. - */ - if (++active_count > 1) { - mutex_unlock(&gov_lock); - return 0; - } - - rc = sysfs_create_group(cpufreq_global_kobject, - &interactive_attr_group); - if (rc) { - mutex_unlock(&gov_lock); - return rc; - } - - idle_notifier_register(&cpufreq_interactive_idle_nb); - cpufreq_register_notifier( - &cpufreq_notifier_block, CPUFREQ_TRANSITION_NOTIFIER); mutex_unlock(&gov_lock); break; @@ -1098,18 +1431,7 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy, up_write(&pcpu->enable_sem); } - if (--active_count > 0) { - mutex_unlock(&gov_lock); - return 0; - } - - cpufreq_unregister_notifier( - &cpufreq_notifier_block, CPUFREQ_TRANSITION_NOTIFIER); - idle_notifier_unregister(&cpufreq_interactive_idle_nb); - sysfs_remove_group(cpufreq_global_kobject, - &interactive_attr_group); mutex_unlock(&gov_lock); - break; case CPUFREQ_GOV_LIMITS: @@ -1119,11 +1441,39 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy, else if (policy->min > policy->cur) __cpufreq_driver_target(policy, policy->min, CPUFREQ_RELATION_L); + for_each_cpu(j, policy->cpus) { + pcpu = &per_cpu(cpuinfo, j); + + down_read(&pcpu->enable_sem); + if (pcpu->governor_enabled == 0) { + up_read(&pcpu->enable_sem); + continue; + } + + spin_lock_irqsave(&pcpu->target_freq_lock, flags); + if (policy->max < pcpu->target_freq) + pcpu->target_freq = policy->max; + else if (policy->min > pcpu->target_freq) + pcpu->target_freq = policy->min; + + spin_unlock_irqrestore(&pcpu->target_freq_lock, flags); + up_read(&pcpu->enable_sem); + } break; } return 0; } +#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE +static +#endif +struct cpufreq_governor cpufreq_gov_interactive = { + .name = "interactive", + .governor = cpufreq_governor_interactive, + .max_transition_latency = 10000000, + .owner = THIS_MODULE, +}; + static void cpufreq_interactive_nop_timer(unsigned long data) { } @@ -1143,10 +1493,10 @@ static int __init cpufreq_interactive_init(void) init_timer(&pcpu->cpu_slack_timer); pcpu->cpu_slack_timer.function = cpufreq_interactive_nop_timer; spin_lock_init(&pcpu->load_lock); + spin_lock_init(&pcpu->target_freq_lock); init_rwsem(&pcpu->enable_sem); } - spin_lock_init(&target_loads_lock); spin_lock_init(&speedchange_cpumask_lock); mutex_init(&gov_lock); speedchange_task =