#include <linux/workqueue.h>
#include <linux/kthread.h>
#include <linux/slab.h>
-#include <linux/kernel_stat.h>
-#include <asm/cputime.h>
#define CREATE_TRACE_POINTS
#include <trace/events/cpufreq_interactive.h>
u64 cputime_speedadj_timestamp;
struct cpufreq_policy *policy;
struct cpufreq_frequency_table *freq_table;
+ spinlock_t target_freq_lock; /*protects target freq */
unsigned int target_freq;
unsigned int floor_freq;
+ unsigned int max_freq;
u64 floor_validate_time;
u64 hispeed_validate_time;
struct rw_semaphore enable_sem;
int boostpulse_duration_val;
/* End time of boost pulse in ktime converted to usecs */
u64 boostpulse_endtime;
+ bool boosted;
/*
* Max additional time to wait in idle, beyond timer_rate, at speeds
* above minimum before wakeup to reduce speed, or -1 if unnecessary.
};
/* For cases where we have single governor instance for system */
-struct cpufreq_interactive_tunables *common_tunables;
+static struct cpufreq_interactive_tunables *common_tunables;
static struct attribute_group *get_sysfs_attr(void);
-static inline cputime64_t get_cpu_idle_time_jiffy(unsigned int cpu,
- cputime64_t *wall)
-{
- u64 idle_time;
- u64 cur_wall_time;
- u64 busy_time;
-
- cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
-
- busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
- busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
- busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
- busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
- busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
- busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
-
- idle_time = cur_wall_time - busy_time;
- if (wall)
- *wall = jiffies_to_usecs(cur_wall_time);
-
- return jiffies_to_usecs(idle_time);
-}
-
-static inline cputime64_t get_cpu_idle_time(
- unsigned int cpu,
- cputime64_t *wall,
- bool io_is_busy)
-{
- u64 idle_time = get_cpu_idle_time_us(cpu, wall);
-
- if (idle_time == -1ULL)
- idle_time = get_cpu_idle_time_jiffy(cpu, wall);
- else if (!io_is_busy)
- idle_time += get_cpu_iowait_time_us(cpu, wall);
-
- return idle_time;
-}
-
static void cpufreq_interactive_timer_resched(
struct cpufreq_interactive_cpuinfo *pcpu)
{
unsigned int loadadjfreq;
unsigned int index;
unsigned long flags;
- bool boosted;
if (!down_read_trylock(&pcpu->enable_sem))
return;
if (WARN_ON_ONCE(!delta_time))
goto rearm;
+ spin_lock_irqsave(&pcpu->target_freq_lock, flags);
do_div(cputime_speedadj, delta_time);
loadadjfreq = (unsigned int)cputime_speedadj * 100;
cpu_load = loadadjfreq / pcpu->target_freq;
- boosted = tunables->boost_val || now < tunables->boostpulse_endtime;
+ tunables->boosted = tunables->boost_val || now < tunables->boostpulse_endtime;
- if (cpu_load >= tunables->go_hispeed_load || boosted) {
+ if (cpu_load >= tunables->go_hispeed_load || tunables->boosted) {
if (pcpu->target_freq < tunables->hispeed_freq) {
new_freq = tunables->hispeed_freq;
} else {
}
} else {
new_freq = choose_freq(pcpu, loadadjfreq);
+ if (new_freq > tunables->hispeed_freq &&
+ pcpu->target_freq < tunables->hispeed_freq)
+ new_freq = tunables->hispeed_freq;
}
if (pcpu->target_freq >= tunables->hispeed_freq &&
trace_cpufreq_interactive_notyet(
data, cpu_load, pcpu->target_freq,
pcpu->policy->cur, new_freq);
+ spin_unlock_irqrestore(&pcpu->target_freq_lock, flags);
goto rearm;
}
if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
new_freq, CPUFREQ_RELATION_L,
- &index))
+ &index)) {
+ spin_unlock_irqrestore(&pcpu->target_freq_lock, flags);
goto rearm;
+ }
new_freq = pcpu->freq_table[index].frequency;
trace_cpufreq_interactive_notyet(
data, cpu_load, pcpu->target_freq,
pcpu->policy->cur, new_freq);
+ spin_unlock_irqrestore(&pcpu->target_freq_lock, flags);
goto rearm;
}
}
* (or the indefinite boost is turned off).
*/
- if (!boosted || new_freq > tunables->hispeed_freq) {
+ if (!tunables->boosted || new_freq > tunables->hispeed_freq) {
pcpu->floor_freq = new_freq;
pcpu->floor_validate_time = now;
}
- if (pcpu->target_freq == new_freq) {
+ if (pcpu->target_freq == new_freq &&
+ pcpu->target_freq <= pcpu->policy->cur) {
trace_cpufreq_interactive_already(
data, cpu_load, pcpu->target_freq,
pcpu->policy->cur, new_freq);
+ spin_unlock_irqrestore(&pcpu->target_freq_lock, flags);
goto rearm_if_notmax;
}
pcpu->policy->cur, new_freq);
pcpu->target_freq = new_freq;
+ spin_unlock_irqrestore(&pcpu->target_freq_lock, flags);
spin_lock_irqsave(&speedchange_cpumask_lock, flags);
cpumask_set_cpu(data, &speedchange_cpumask);
spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
return 0;
}
-static void cpufreq_interactive_boost(void)
+static void cpufreq_interactive_boost(struct cpufreq_interactive_tunables *tunables)
{
int i;
int anyboost = 0;
- unsigned long flags;
+ unsigned long flags[2];
struct cpufreq_interactive_cpuinfo *pcpu;
- struct cpufreq_interactive_tunables *tunables;
- spin_lock_irqsave(&speedchange_cpumask_lock, flags);
+ tunables->boosted = true;
+
+ spin_lock_irqsave(&speedchange_cpumask_lock, flags[0]);
for_each_online_cpu(i) {
pcpu = &per_cpu(cpuinfo, i);
- tunables = pcpu->policy->governor_data;
+ if (tunables != pcpu->policy->governor_data)
+ continue;
+ spin_lock_irqsave(&pcpu->target_freq_lock, flags[1]);
if (pcpu->target_freq < tunables->hispeed_freq) {
pcpu->target_freq = tunables->hispeed_freq;
cpumask_set_cpu(i, &speedchange_cpumask);
ktime_to_us(ktime_get());
anyboost = 1;
}
-
- /*
- * Set floor freq and (re)start timer for when last
- * validated.
- */
-
- pcpu->floor_freq = tunables->hispeed_freq;
- pcpu->floor_validate_time = ktime_to_us(ktime_get());
+ spin_unlock_irqrestore(&pcpu->target_freq_lock, flags[1]);
}
- spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
+ spin_unlock_irqrestore(&speedchange_cpumask_lock, flags[0]);
if (anyboost)
wake_up_process(speedchange_task);
ret += sprintf(buf + ret, "%u%s", tunables->target_loads[i],
i & 0x1 ? ":" : " ");
- ret += sprintf(buf + --ret, "\n");
+ sprintf(buf + ret - 1, "\n");
spin_unlock_irqrestore(&tunables->target_loads_lock, flags);
return ret;
}
tunables->above_hispeed_delay[i],
i & 0x1 ? ":" : " ");
- ret += sprintf(buf + --ret, "\n");
+ sprintf(buf + ret - 1, "\n");
spin_unlock_irqrestore(&tunables->above_hispeed_delay_lock, flags);
return ret;
}
const char *buf, size_t count)
{
int ret;
- unsigned long val;
+ unsigned long val, val_round;
ret = strict_strtoul(buf, 0, &val);
if (ret < 0)
return ret;
- tunables->timer_rate = val;
+
+ val_round = jiffies_to_usecs(usecs_to_jiffies(val));
+ if (val != val_round)
+ pr_warn("timer_rate not aligned to jiffy. Rounded up to %lu\n",
+ val_round);
+
+ tunables->timer_rate = val_round;
return count;
}
if (tunables->boost_val) {
trace_cpufreq_interactive_boost("on");
- cpufreq_interactive_boost();
+ if (!tunables->boosted)
+ cpufreq_interactive_boost(tunables);
} else {
+ tunables->boostpulse_endtime = ktime_to_us(ktime_get());
trace_cpufreq_interactive_unboost("off");
}
tunables->boostpulse_endtime = ktime_to_us(ktime_get()) +
tunables->boostpulse_duration_val;
trace_cpufreq_interactive_boost("pulse");
- cpufreq_interactive_boost();
+ if (!tunables->boosted)
+ cpufreq_interactive_boost(tunables);
return count;
}
struct cpufreq_interactive_cpuinfo *pcpu;
struct cpufreq_frequency_table *freq_table;
struct cpufreq_interactive_tunables *tunables;
+ unsigned long flags;
if (have_governor_per_policy())
tunables = policy->governor_data;
return -ENOMEM;
}
- rc = sysfs_create_group(get_governor_parent_kobj(policy),
- get_sysfs_attr());
- if (rc) {
- kfree(tunables);
- return rc;
- }
-
tunables->usage_count = 1;
tunables->above_hispeed_delay = default_above_hispeed_delay;
tunables->nabove_hispeed_delay =
spin_lock_init(&tunables->target_loads_lock);
spin_lock_init(&tunables->above_hispeed_delay_lock);
+ policy->governor_data = tunables;
+ if (!have_governor_per_policy()) {
+ common_tunables = tunables;
+ WARN_ON(cpufreq_get_global_kobject());
+ }
+
+ rc = sysfs_create_group(get_governor_parent_kobj(policy),
+ get_sysfs_attr());
+ if (rc) {
+ kfree(tunables);
+ policy->governor_data = NULL;
+ if (!have_governor_per_policy()) {
+ common_tunables = NULL;
+ cpufreq_put_global_kobject();
+ }
+ return rc;
+ }
+
if (!policy->governor->initialized) {
idle_notifier_register(&cpufreq_interactive_idle_nb);
cpufreq_register_notifier(&cpufreq_notifier_block,
CPUFREQ_TRANSITION_NOTIFIER);
}
- policy->governor_data = tunables;
- if (!have_governor_per_policy())
- common_tunables = tunables;
-
break;
case CPUFREQ_GOV_POLICY_EXIT:
sysfs_remove_group(get_governor_parent_kobj(policy),
get_sysfs_attr());
+
+ if (!have_governor_per_policy())
+ cpufreq_put_global_kobject();
+
kfree(tunables);
common_tunables = NULL;
}
ktime_to_us(ktime_get());
pcpu->hispeed_validate_time =
pcpu->floor_validate_time;
+ pcpu->max_freq = policy->max;
down_write(&pcpu->enable_sem);
del_timer_sync(&pcpu->cpu_timer);
del_timer_sync(&pcpu->cpu_slack_timer);
for_each_cpu(j, policy->cpus) {
pcpu = &per_cpu(cpuinfo, j);
- /* hold write semaphore to avoid race */
- down_write(&pcpu->enable_sem);
+ down_read(&pcpu->enable_sem);
if (pcpu->governor_enabled == 0) {
- up_write(&pcpu->enable_sem);
+ up_read(&pcpu->enable_sem);
continue;
}
- /* update target_freq firstly */
+ spin_lock_irqsave(&pcpu->target_freq_lock, flags);
if (policy->max < pcpu->target_freq)
pcpu->target_freq = policy->max;
else if (policy->min > pcpu->target_freq)
pcpu->target_freq = policy->min;
- /* Reschedule timer.
+ spin_unlock_irqrestore(&pcpu->target_freq_lock, flags);
+ up_read(&pcpu->enable_sem);
+
+ /* Reschedule timer only if policy->max is raised.
* Delete the timers, else the timer callback may
* return without re-arm the timer when failed
* acquire the semaphore. This race may cause timer
* stopped unexpectedly.
*/
- del_timer_sync(&pcpu->cpu_timer);
- del_timer_sync(&pcpu->cpu_slack_timer);
- cpufreq_interactive_timer_start(tunables, j);
- up_write(&pcpu->enable_sem);
+
+ if (policy->max > pcpu->max_freq) {
+ down_write(&pcpu->enable_sem);
+ del_timer_sync(&pcpu->cpu_timer);
+ del_timer_sync(&pcpu->cpu_slack_timer);
+ cpufreq_interactive_timer_start(tunables, j);
+ up_write(&pcpu->enable_sem);
+ }
+
+ pcpu->max_freq = policy->max;
}
break;
}
init_timer(&pcpu->cpu_slack_timer);
pcpu->cpu_slack_timer.function = cpufreq_interactive_nop_timer;
spin_lock_init(&pcpu->load_lock);
+ spin_lock_init(&pcpu->target_freq_lock);
init_rwsem(&pcpu->enable_sem);
}
projects / firefly-linux-kernel-4.4.55.git / blobdiff