cpufreq: interactive: pin timers to associated CPU

Helps avoid waking up other CPUs to react to activity on the local CPU.

Change-Id: Ife272aaa7916894a437705d44521b1a1693fbe8e
Signed-off-by: Todd Poynor <toddpoynor@google.com>

diff --git a/drivers/cpufreq/cpufreq_interactive.c b/drivers/cpufreq/cpufreq_interactive.c
index 16bd23be2495bdebd46a5aff00c5df1a64524d3f..e53eae2214990186bef0026b8bec9fb09fca0f6d 100644 (file)
--- a/drivers/cpufreq/cpufreq_interactive.c
+++ b/drivers/cpufreq/cpufreq_interactive.c
@@ -42,8 +42,6 @@ struct cpufreq_interactive_cpuinfo {
        int timer_idlecancel;
        u64 time_in_idle;
        u64 idle_exit_time;
        int timer_idlecancel;
        u64 time_in_idle;
        u64 idle_exit_time;

-       u64 timer_run_time;
-       int idling;

        u64 target_set_time;
        u64 target_set_time_in_idle;
        struct cpufreq_policy *policy;
        u64 target_set_time;
        u64 target_set_time_in_idle;
        struct cpufreq_policy *policy;


@@ -109,6 +107,7 @@ struct cpufreq_governor cpufreq_gov_interactive = {
 
 static void cpufreq_interactive_timer(unsigned long data)
 {
 
 static void cpufreq_interactive_timer(unsigned long data)
 {

+       u64 now;

        unsigned int delta_idle;
        unsigned int delta_time;
        int cpu_load;
        unsigned int delta_idle;
        unsigned int delta_time;
        int cpu_load;


@@ -127,26 +126,11 @@ static void cpufreq_interactive_timer(unsigned long data)
        if (!pcpu->governor_enabled)
                goto exit;
 
        if (!pcpu->governor_enabled)
                goto exit;
 

-       /*
-        * Once pcpu->timer_run_time is updated to >= pcpu->idle_exit_time,
-        * this lets idle exit know the current idle time sample has
-        * been processed, and idle exit can generate a new sample and
-        * re-arm the timer.  This prevents a concurrent idle
-        * exit on that CPU from writing a new set of info at the same time
-        * the timer function runs (the timer function can't use that info
-        * until more time passes).
-        */

        time_in_idle = pcpu->time_in_idle;
        idle_exit_time = pcpu->idle_exit_time;
        time_in_idle = pcpu->time_in_idle;
        idle_exit_time = pcpu->idle_exit_time;

-       now_idle = get_cpu_idle_time_us(data, &pcpu->timer_run_time);
-       smp_wmb();
-
-       /* If we raced with cancelling a timer, skip. */
-       if (!idle_exit_time)
-               goto exit;
-
+       now_idle = get_cpu_idle_time_us(data, &now);

        delta_idle = (unsigned int)(now_idle - time_in_idle);
        delta_idle = (unsigned int)(now_idle - time_in_idle);

-       delta_time = (unsigned int)(pcpu->timer_run_time - idle_exit_time);
+       delta_time = (unsigned int)(now - idle_exit_time);

 
        /*
         * If timer ran less than 1ms after short-term sample started, retry.
 
        /*
         * If timer ran less than 1ms after short-term sample started, retry.


@@ -160,8 +144,7 @@ static void cpufreq_interactive_timer(unsigned long data)
                cpu_load = 100 * (delta_time - delta_idle) / delta_time;
 
        delta_idle = (unsigned int)(now_idle - pcpu->target_set_time_in_idle);
                cpu_load = 100 * (delta_time - delta_idle) / delta_time;
 
        delta_idle = (unsigned int)(now_idle - pcpu->target_set_time_in_idle);

-       delta_time = (unsigned int)(pcpu->timer_run_time -
-                                   pcpu->target_set_time);
+       delta_time = (unsigned int)(now - pcpu->target_set_time);

 
        if ((delta_time == 0) || (delta_idle > delta_time))
                load_since_change = 0;
 
        if ((delta_time == 0) || (delta_idle > delta_time))
                load_since_change = 0;


@@ -189,7 +172,7 @@ static void cpufreq_interactive_timer(unsigned long data)
 
                        if (pcpu->target_freq == hispeed_freq &&
                            new_freq > hispeed_freq &&
 
                        if (pcpu->target_freq == hispeed_freq &&
                            new_freq > hispeed_freq &&

-                           pcpu->timer_run_time - pcpu->hispeed_validate_time
+                           now - pcpu->hispeed_validate_time

                            < above_hispeed_delay_val) {
                                trace_cpufreq_interactive_notyet(data, cpu_load,
                                                                 pcpu->target_freq,
                            < above_hispeed_delay_val) {
                                trace_cpufreq_interactive_notyet(data, cpu_load,
                                                                 pcpu->target_freq,


@@ -202,7 +185,7 @@ static void cpufreq_interactive_timer(unsigned long data)
        }
 
        if (new_freq <= hispeed_freq)
        }
 
        if (new_freq <= hispeed_freq)

-               pcpu->hispeed_validate_time = pcpu->timer_run_time;
+               pcpu->hispeed_validate_time = now;

 
        if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
                                           new_freq, CPUFREQ_RELATION_H,
 
        if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
                                           new_freq, CPUFREQ_RELATION_H,


@@ -219,8 +202,7 @@ static void cpufreq_interactive_timer(unsigned long data)
         * floor frequency for the minimum sample time since last validated.
         */
        if (new_freq < pcpu->floor_freq) {
         * floor frequency for the minimum sample time since last validated.
         */
        if (new_freq < pcpu->floor_freq) {

-               if (pcpu->timer_run_time - pcpu->floor_validate_time
-                   < min_sample_time) {
+               if (now - pcpu->floor_validate_time < min_sample_time) {

                        trace_cpufreq_interactive_notyet(data, cpu_load,
                                         pcpu->target_freq, new_freq);
                        goto rearm;
                        trace_cpufreq_interactive_notyet(data, cpu_load,
                                         pcpu->target_freq, new_freq);
                        goto rearm;


@@ -228,7 +210,7 @@ static void cpufreq_interactive_timer(unsigned long data)
        }
 
        pcpu->floor_freq = new_freq;
        }
 
        pcpu->floor_freq = new_freq;

-       pcpu->floor_validate_time = pcpu->timer_run_time;
+       pcpu->floor_validate_time = now;

 
        if (pcpu->target_freq == new_freq) {
                trace_cpufreq_interactive_already(data, cpu_load,
 
        if (pcpu->target_freq == new_freq) {
                trace_cpufreq_interactive_already(data, cpu_load,


@@ -239,7 +221,7 @@ static void cpufreq_interactive_timer(unsigned long data)
        trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
                                         new_freq);
        pcpu->target_set_time_in_idle = now_idle;
        trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
                                         new_freq);
        pcpu->target_set_time_in_idle = now_idle;

-       pcpu->target_set_time = pcpu->timer_run_time;
+       pcpu->target_set_time = now;

 
        pcpu->target_freq = new_freq;
        spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 
        pcpu->target_freq = new_freq;
        spin_lock_irqsave(&speedchange_cpumask_lock, flags);


@@ -258,23 +240,16 @@ rearm_if_notmax:
 rearm:
        if (!timer_pending(&pcpu->cpu_timer)) {
                /*
 rearm:
        if (!timer_pending(&pcpu->cpu_timer)) {
                /*

-                * If already at min: if that CPU is idle, don't set timer.
-                * Else cancel the timer if that CPU goes idle.  We don't
-                * need to re-evaluate speed until the next idle exit.
+                * If already at min, cancel the timer if that CPU goes idle.
+                * We don't need to re-evaluate speed until the next idle exit.

                 */
                 */

-               if (pcpu->target_freq == pcpu->policy->min) {
-                       smp_rmb();
-
-                       if (pcpu->idling)
-                               goto exit;
-
+               if (pcpu->target_freq == pcpu->policy->min)

                        pcpu->timer_idlecancel = 1;
                        pcpu->timer_idlecancel = 1;

-               }

 
                pcpu->time_in_idle = get_cpu_idle_time_us(
                        data, &pcpu->idle_exit_time);
 
                pcpu->time_in_idle = get_cpu_idle_time_us(
                        data, &pcpu->idle_exit_time);

-               mod_timer(&pcpu->cpu_timer,
-                         jiffies + usecs_to_jiffies(timer_rate));
+               mod_timer_pinned(&pcpu->cpu_timer,
+                                jiffies + usecs_to_jiffies(timer_rate));

        }
 
 exit:
        }
 
 exit:


@@ -290,8 +265,6 @@ static void cpufreq_interactive_idle_start(void)
        if (!pcpu->governor_enabled)
                return;
 
        if (!pcpu->governor_enabled)
                return;
 

-       pcpu->idling = 1;
-       smp_wmb();

        pending = timer_pending(&pcpu->cpu_timer);
 
        if (pcpu->target_freq != pcpu->policy->min) {
        pending = timer_pending(&pcpu->cpu_timer);
 
        if (pcpu->target_freq != pcpu->policy->min) {


@@ -308,8 +281,9 @@ static void cpufreq_interactive_idle_start(void)
                        pcpu->time_in_idle = get_cpu_idle_time_us(
                                smp_processor_id(), &pcpu->idle_exit_time);
                        pcpu->timer_idlecancel = 0;
                        pcpu->time_in_idle = get_cpu_idle_time_us(
                                smp_processor_id(), &pcpu->idle_exit_time);
                        pcpu->timer_idlecancel = 0;

-                       mod_timer(&pcpu->cpu_timer,
-                                 jiffies + usecs_to_jiffies(timer_rate));
+                       mod_timer_pinned(
+                               &pcpu->cpu_timer,
+                               jiffies + usecs_to_jiffies(timer_rate));

                }
 #endif
        } else {
                }
 #endif
        } else {


@@ -321,12 +295,6 @@ static void cpufreq_interactive_idle_start(void)
                 */
                if (pending && pcpu->timer_idlecancel) {
                        del_timer(&pcpu->cpu_timer);
                 */
                if (pending && pcpu->timer_idlecancel) {
                        del_timer(&pcpu->cpu_timer);

-                       /*
-                        * Ensure last timer run time is after current idle
-                        * sample start time, so next idle exit will always
-                        * start a new idle sampling period.
-                        */
-                       pcpu->idle_exit_time = 0;

                        pcpu->timer_idlecancel = 0;
                }
        }
                        pcpu->timer_idlecancel = 0;
                }
        }


@@ -341,29 +309,15 @@ static void cpufreq_interactive_idle_end(void)
        if (!pcpu->governor_enabled)
                return;
 
        if (!pcpu->governor_enabled)
                return;
 

-       pcpu->idling = 0;
-       smp_wmb();
-
-       /*
-        * Arm the timer for 1-2 ticks later if not already, and if the timer
-        * function has already processed the previous load sampling
-        * interval.  (If the timer is not pending but has not processed
-        * the previous interval, it is probably racing with us on another
-        * CPU.  Let it compute load based on the previous sample and then
-        * re-arm the timer for another interval when it's done, rather
-        * than updating the interval start time to be "now", which doesn't
-        * give the timer function enough time to make a decision on this
-        * run.)
-        */
-       if (timer_pending(&pcpu->cpu_timer) == 0 &&
-           pcpu->timer_run_time >= pcpu->idle_exit_time &&
-           pcpu->governor_enabled) {
+       /* Arm the timer for 1-2 ticks later if not already. */
+       if (!timer_pending(&pcpu->cpu_timer)) {

                pcpu->time_in_idle =
                        get_cpu_idle_time_us(smp_processor_id(),
                                             &pcpu->idle_exit_time);
                pcpu->timer_idlecancel = 0;
                pcpu->time_in_idle =
                        get_cpu_idle_time_us(smp_processor_id(),
                                             &pcpu->idle_exit_time);
                pcpu->timer_idlecancel = 0;

-               mod_timer(&pcpu->cpu_timer,
-                         jiffies + usecs_to_jiffies(timer_rate));
+               mod_timer_pinned(
+                       &pcpu->cpu_timer,
+                       jiffies + usecs_to_jiffies(timer_rate));

        }
 
 }
        }
 
 }


@@ -673,6 +627,8 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 
                freq_table =
                        cpufreq_frequency_get_table(policy->cpu);
 
                freq_table =
                        cpufreq_frequency_get_table(policy->cpu);

+               if (!hispeed_freq)
+                       hispeed_freq = policy->max;

 
                for_each_cpu(j, policy->cpus) {
                        pcpu = &per_cpu(cpuinfo, j);
 
                for_each_cpu(j, policy->cpus) {
                        pcpu = &per_cpu(cpuinfo, j);


@@ -689,11 +645,11 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
                                pcpu->target_set_time;
                        pcpu->governor_enabled = 1;
                        smp_wmb();
                                pcpu->target_set_time;
                        pcpu->governor_enabled = 1;
                        smp_wmb();

+                       pcpu->cpu_timer.expires =
+                               jiffies + usecs_to_jiffies(timer_rate);
+                       add_timer_on(&pcpu->cpu_timer, j);

                }
 
                }
 

-               if (!hispeed_freq)
-                       hispeed_freq = policy->max;
-

                /*
                 * Do not register the idle hook and create sysfs
                 * entries if we have already done so.
                /*
                 * Do not register the idle hook and create sysfs
                 * entries if we have already done so.


@@ -715,14 +671,6 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
                        pcpu->governor_enabled = 0;
                        smp_wmb();
                        del_timer_sync(&pcpu->cpu_timer);
                        pcpu->governor_enabled = 0;
                        smp_wmb();
                        del_timer_sync(&pcpu->cpu_timer);

-
-                       /*
-                        * Reset idle exit time since we may cancel the timer
-                        * before it can run after the last idle exit time,
-                        * to avoid tripping the check in idle exit for a timer
-                        * that is trying to run.
-                        */
-                       pcpu->idle_exit_time = 0;

                }
 
                if (atomic_dec_return(&active_count) > 0)
                }
 
                if (atomic_dec_return(&active_count) > 0)