drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *
   7  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   8  *      Added handling for CPU hotplug
   9  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  10  *      Fix handling for CPU hotplug -- affected CPUs
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License version 2 as
  14  * published by the Free Software Foundation.
  15  *
  16  */
  17
  18 #include <linux/kernel.h>
  19 #include <linux/module.h>
  20 #include <linux/init.h>
  21 #include <linux/notifier.h>
  22 #include <linux/cpufreq.h>
  23 #include <linux/delay.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/spinlock.h>
  26 #include <linux/device.h>
  27 #include <linux/slab.h>
  28 #include <linux/cpu.h>
  29 #include <linux/completion.h>
  30 #include <linux/mutex.h>
  31
  32 #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_CORE, \
  33                                                 "cpufreq-core", msg)
  34
  35 /**
  36  * The "cpufreq driver" - the arch- or hardware-dependent low
  37  * level driver of CPUFreq support, and its spinlock. This lock
  38  * also protects the cpufreq_cpu_data array.
  39  */
  40 static struct cpufreq_driver *cpufreq_driver;
  41 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  42 #ifdef CONFIG_HOTPLUG_CPU
  43 /* This one keeps track of the previously set governor of a removed CPU */
  44 static DEFINE_PER_CPU(struct cpufreq_governor *, cpufreq_cpu_governor);
  45 #endif
  46 static DEFINE_SPINLOCK(cpufreq_driver_lock);
  47
  48 /*
  49  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  50  * all cpufreq/hotplug/workqueue/etc related lock issues.
  51  *
  52  * The rules for this semaphore:
  53  * - Any routine that wants to read from the policy structure will
  54  *   do a down_read on this semaphore.
  55  * - Any routine that will write to the policy structure and/or may take away
  56  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  57  *   mode before doing so.
  58  *
  59  * Additional rules:
  60  * - All holders of the lock should check to make sure that the CPU they
  61  *   are concerned with are online after they get the lock.
  62  * - Governor routines that can be called in cpufreq hotplug path should not
  63  *   take this sem as top level hotplug notifier handler takes this.
  64  * - Lock should not be held across
  65  *     __cpufreq_governor(data, CPUFREQ_GOV_STOP);
  66  */
  67 static DEFINE_PER_CPU(int, policy_cpu);
  68 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  69
  70 #define lock_policy_rwsem(mode, cpu)                                    \
  71 int lock_policy_rwsem_##mode                                            \
  72 (int cpu)                                                               \
  73 {                                                                       \
  74         int policy_cpu = per_cpu(policy_cpu, cpu);                      \
  75         BUG_ON(policy_cpu == -1);                                       \
  76         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  77         if (unlikely(!cpu_online(cpu))) {                               \
  78                 up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));      \
  79                 return -1;                                              \
  80         }                                                               \
  81                                                                         \
  82         return 0;                                                       \
  83 }
  84
  85 lock_policy_rwsem(read, cpu);
  86 EXPORT_SYMBOL_GPL(lock_policy_rwsem_read);
  87
  88 lock_policy_rwsem(write, cpu);
  89 EXPORT_SYMBOL_GPL(lock_policy_rwsem_write);
  90
  91 void unlock_policy_rwsem_read(int cpu)
  92 {
  93         int policy_cpu = per_cpu(policy_cpu, cpu);
  94         BUG_ON(policy_cpu == -1);
  95         up_read(&per_cpu(cpu_policy_rwsem, policy_cpu));
  96 }
  97 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_read);
  98
  99 void unlock_policy_rwsem_write(int cpu)
 100 {
 101         int policy_cpu = per_cpu(policy_cpu, cpu);
 102         BUG_ON(policy_cpu == -1);
 103         up_write(&per_cpu(cpu_policy_rwsem, policy_cpu));
 104 }
 105 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_write);
 106
 107
 108 /* internal prototypes */
 109 static int __cpufreq_governor(struct cpufreq_policy *policy,
 110                 unsigned int event);
 111 static unsigned int __cpufreq_get(unsigned int cpu);
 112 static void handle_update(struct work_struct *work);
 113
 114 /**
 115  * Two notifier lists: the "policy" list is involved in the
 116  * validation process for a new CPU frequency policy; the
 117  * "transition" list for kernel code that needs to handle
 118  * changes to devices when the CPU clock speed changes.
 119  * The mutex locks both lists.
 120  */
 121 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 122 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 123
 124 static bool init_cpufreq_transition_notifier_list_called;
 125 static int __init init_cpufreq_transition_notifier_list(void)
 126 {
 127         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 128         init_cpufreq_transition_notifier_list_called = true;
 129         return 0;
 130 }
 131 pure_initcall(init_cpufreq_transition_notifier_list);
 132
 133 static LIST_HEAD(cpufreq_governor_list);
 134 static DEFINE_MUTEX(cpufreq_governor_mutex);
 135
 136 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 137 {
 138         struct cpufreq_policy *data;
 139         unsigned long flags;
 140
 141         if (cpu >= nr_cpu_ids)
 142                 goto err_out;
 143
 144         /* get the cpufreq driver */
 145         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 146
 147         if (!cpufreq_driver)
 148                 goto err_out_unlock;
 149
 150         if (!try_module_get(cpufreq_driver->owner))
 151                 goto err_out_unlock;
 152
 153
 154         /* get the CPU */
 155         data = per_cpu(cpufreq_cpu_data, cpu);
 156
 157         if (!data)
 158                 goto err_out_put_module;
 159
 160         if (!kobject_get(&data->kobj))
 161                 goto err_out_put_module;
 162
 163         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 164         return data;
 165
 166 err_out_put_module:
 167         module_put(cpufreq_driver->owner);
 168 err_out_unlock:
 169         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 170 err_out:
 171         return NULL;
 172 }
 173 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 174
 175
 176 void cpufreq_cpu_put(struct cpufreq_policy *data)
 177 {
 178         kobject_put(&data->kobj);
 179         module_put(cpufreq_driver->owner);
 180 }
 181 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 182
 183
 184 /*********************************************************************
 185  *                     UNIFIED DEBUG HELPERS                         *
 186  *********************************************************************/
 187 #ifdef CONFIG_CPU_FREQ_DEBUG
 188
 189 /* what part(s) of the CPUfreq subsystem are debugged? */
 190 static unsigned int debug;
 191
 192 /* is the debug output ratelimit'ed using printk_ratelimit? User can
 193  * set or modify this value.
 194  */
 195 static unsigned int debug_ratelimit = 1;
 196
 197 /* is the printk_ratelimit'ing enabled? It's enabled after a successful
 198  * loading of a cpufreq driver, temporarily disabled when a new policy
 199  * is set, and disabled upon cpufreq driver removal
 200  */
 201 static unsigned int disable_ratelimit = 1;
 202 static DEFINE_SPINLOCK(disable_ratelimit_lock);
 203
 204 static void cpufreq_debug_enable_ratelimit(void)
 205 {
 206         unsigned long flags;
 207
 208         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 209         if (disable_ratelimit)
 210                 disable_ratelimit--;
 211         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 212 }
 213
 214 static void cpufreq_debug_disable_ratelimit(void)
 215 {
 216         unsigned long flags;
 217
 218         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 219         disable_ratelimit++;
 220         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 221 }
 222
 223 void cpufreq_debug_printk(unsigned int type, const char *prefix,
 224                         const char *fmt, ...)
 225 {
 226         char s[256];
 227         va_list args;
 228         unsigned int len;
 229         unsigned long flags;
 230
 231         WARN_ON(!prefix);
 232         if (type & debug) {
 233                 spin_lock_irqsave(&disable_ratelimit_lock, flags);
 234                 if (!disable_ratelimit && debug_ratelimit
 235                                         && !printk_ratelimit()) {
 236                         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 237                         return;
 238                 }
 239                 spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 240
 241                 len = snprintf(s, 256, KERN_DEBUG "%s: ", prefix);
 242
 243                 va_start(args, fmt);
 244                 len += vsnprintf(&s[len], (256 - len), fmt, args);
 245                 va_end(args);
 246
 247                 printk(s);
 248
 249                 WARN_ON(len < 5);
 250         }
 251 }
 252 EXPORT_SYMBOL(cpufreq_debug_printk);
 253
 254
 255 module_param(debug, uint, 0644);
 256 MODULE_PARM_DESC(debug, "CPUfreq debugging: add 1 to debug core,"
 257                         " 2 to debug drivers, and 4 to debug governors.");
 258
 259 module_param(debug_ratelimit, uint, 0644);
 260 MODULE_PARM_DESC(debug_ratelimit, "CPUfreq debugging:"
 261                                         " set to 0 to disable ratelimiting.");
 262
 263 #else /* !CONFIG_CPU_FREQ_DEBUG */
 264
 265 static inline void cpufreq_debug_enable_ratelimit(void) { return; }
 266 static inline void cpufreq_debug_disable_ratelimit(void) { return; }
 267
 268 #endif /* CONFIG_CPU_FREQ_DEBUG */
 269
 270
 271 /*********************************************************************
 272  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 273  *********************************************************************/
 274
 275 /**
 276  * adjust_jiffies - adjust the system "loops_per_jiffy"
 277  *
 278  * This function alters the system "loops_per_jiffy" for the clock
 279  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 280  * systems as each CPU might be scaled differently. So, use the arch
 281  * per-CPU loops_per_jiffy value wherever possible.
 282  */
 283 #ifndef CONFIG_SMP
 284 static unsigned long l_p_j_ref;
 285 static unsigned int  l_p_j_ref_freq;
 286
 287 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 288 {
 289         if (ci->flags & CPUFREQ_CONST_LOOPS)
 290                 return;
 291
 292         if (!l_p_j_ref_freq) {
 293                 l_p_j_ref = loops_per_jiffy;
 294                 l_p_j_ref_freq = ci->old;
 295                 dprintk("saving %lu as reference value for loops_per_jiffy; "
 296                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 297         }
 298         if ((val == CPUFREQ_PRECHANGE  && ci->old < ci->new) ||
 299             (val == CPUFREQ_POSTCHANGE && ci->old > ci->new) ||
 300             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 301                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 302                                                                 ci->new);
 303                 dprintk("scaling loops_per_jiffy to %lu "
 304                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 305         }
 306 }
 307 #else
 308 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 309 {
 310         return;
 311 }
 312 #endif
 313
 314
 315 /**
 316  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 317  * on frequency transition.
 318  *
 319  * This function calls the transition notifiers and the "adjust_jiffies"
 320  * function. It is called twice on all CPU frequency changes that have
 321  * external effects.
 322  */
 323 void cpufreq_notify_transition(struct cpufreq_freqs *freqs, unsigned int state)
 324 {
 325         struct cpufreq_policy *policy;
 326
 327         BUG_ON(irqs_disabled());
 328
 329         freqs->flags = cpufreq_driver->flags;
 330         dprintk("notification %u of frequency transition to %u kHz\n",
 331                 state, freqs->new);
 332
 333         policy = per_cpu(cpufreq_cpu_data, freqs->cpu);
 334         switch (state) {
 335
 336         case CPUFREQ_PRECHANGE:
 337                 /* detect if the driver reported a value as "old frequency"
 338                  * which is not equal to what the cpufreq core thinks is
 339                  * "old frequency".
 340                  */
 341                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 342                         if ((policy) && (policy->cpu == freqs->cpu) &&
 343                             (policy->cur) && (policy->cur != freqs->old)) {
 344                                 dprintk("Warning: CPU frequency is"
 345                                         " %u, cpufreq assumed %u kHz.\n",
 346                                         freqs->old, policy->cur);
 347                                 freqs->old = policy->cur;
 348                         }
 349                 }
 350                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 351                                 CPUFREQ_PRECHANGE, freqs);
 352                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 353                 break;
 354
 355         case CPUFREQ_POSTCHANGE:
 356                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 357                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 358                                 CPUFREQ_POSTCHANGE, freqs);
 359                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 360                         policy->cur = freqs->new;
 361                 break;
 362         }
 363 }
 364 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 365
 366
 367
 368 /*********************************************************************
 369  *                          SYSFS INTERFACE                          *
 370  *********************************************************************/
 371
 372 static struct cpufreq_governor *__find_governor(const char *str_governor)
 373 {
 374         struct cpufreq_governor *t;
 375
 376         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 377                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 378                         return t;
 379
 380         return NULL;
 381 }
 382
 383 /**
 384  * cpufreq_parse_governor - parse a governor string
 385  */
 386 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 387                                 struct cpufreq_governor **governor)
 388 {
 389         int err = -EINVAL;
 390
 391         if (!cpufreq_driver)
 392                 goto out;
 393
 394         if (cpufreq_driver->setpolicy) {
 395                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 396                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 397                         err = 0;
 398                 } else if (!strnicmp(str_governor, "powersave",
 399                                                 CPUFREQ_NAME_LEN)) {
 400                         *policy = CPUFREQ_POLICY_POWERSAVE;
 401                         err = 0;
 402                 }
 403         } else if (cpufreq_driver->target) {
 404                 struct cpufreq_governor *t;
 405
 406                 mutex_lock(&cpufreq_governor_mutex);
 407
 408                 t = __find_governor(str_governor);
 409
 410                 if (t == NULL) {
 411                         char *name = kasprintf(GFP_KERNEL, "cpufreq_%s",
 412                                                                 str_governor);
 413
 414                         if (name) {
 415                                 int ret;
 416
 417                                 mutex_unlock(&cpufreq_governor_mutex);
 418                                 ret = request_module("%s", name);
 419                                 mutex_lock(&cpufreq_governor_mutex);
 420
 421                                 if (ret == 0)
 422                                         t = __find_governor(str_governor);
 423                         }
 424
 425                         kfree(name);
 426                 }
 427
 428                 if (t != NULL) {
 429                         *governor = t;
 430                         err = 0;
 431                 }
 432
 433                 mutex_unlock(&cpufreq_governor_mutex);
 434         }
 435 out:
 436         return err;
 437 }
 438
 439
 440 /**
 441  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 442  * print out cpufreq information
 443  *
 444  * Write out information from cpufreq_driver->policy[cpu]; object must be
 445  * "unsigned int".
 446  */
 447
 448 #define show_one(file_name, object)                     \
 449 static ssize_t show_##file_name                         \
 450 (struct cpufreq_policy *policy, char *buf)              \
 451 {                                                       \
 452         return sprintf(buf, "%u\n", policy->object);    \
 453 }
 454
 455 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 456 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 457 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 458 show_one(scaling_min_freq, min);
 459 show_one(scaling_max_freq, max);
 460 show_one(scaling_cur_freq, cur);
 461
 462 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 463                                 struct cpufreq_policy *policy);
 464
 465 /**
 466  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 467  */
 468 #define store_one(file_name, object)                    \
 469 static ssize_t store_##file_name                                        \
 470 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 471 {                                                                       \
 472         unsigned int ret = -EINVAL;                                     \
 473         struct cpufreq_policy new_policy;                               \
 474                                                                         \
 475         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 476         if (ret)                                                        \
 477                 return -EINVAL;                                         \
 478                                                                         \
 479         ret = sscanf(buf, "%u", &new_policy.object);                    \
 480         if (ret != 1)                                                   \
 481                 return -EINVAL;                                         \
 482                                                                         \
 483         ret = __cpufreq_set_policy(policy, &new_policy);                \
 484         policy->user_policy.object = policy->object;                    \
 485                                                                         \
 486         return ret ? ret : count;                                       \
 487 }
 488
 489 store_one(scaling_min_freq, min);
 490 store_one(scaling_max_freq, max);
 491
 492 /**
 493  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 494  */
 495 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 496                                         char *buf)
 497 {
 498         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 499         if (!cur_freq)
 500                 return sprintf(buf, "<unknown>");
 501         return sprintf(buf, "%u\n", cur_freq);
 502 }
 503
 504
 505 /**
 506  * show_scaling_governor - show the current policy for the specified CPU
 507  */
 508 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 509 {
 510         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 511                 return sprintf(buf, "powersave\n");
 512         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 513                 return sprintf(buf, "performance\n");
 514         else if (policy->governor)
 515                 return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n",
 516                                 policy->governor->name);
 517         return -EINVAL;
 518 }
 519
 520
 521 /**
 522  * store_scaling_governor - store policy for the specified CPU
 523  */
 524 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 525                                         const char *buf, size_t count)
 526 {
 527         unsigned int ret = -EINVAL;
 528         char    str_governor[16];
 529         struct cpufreq_policy new_policy;
 530
 531         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 532         if (ret)
 533                 return ret;
 534
 535         ret = sscanf(buf, "%15s", str_governor);
 536         if (ret != 1)
 537                 return -EINVAL;
 538
 539         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 540                                                 &new_policy.governor))
 541                 return -EINVAL;
 542
 543         /* Do not use cpufreq_set_policy here or the user_policy.max
 544            will be wrongly overridden */
 545         ret = __cpufreq_set_policy(policy, &new_policy);
 546
 547         policy->user_policy.policy = policy->policy;
 548         policy->user_policy.governor = policy->governor;
 549
 550         if (ret)
 551                 return ret;
 552         else
 553                 return count;
 554 }
 555
 556 /**
 557  * show_scaling_driver - show the cpufreq driver currently loaded
 558  */
 559 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 560 {
 561         return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", cpufreq_driver->name);
 562 }
 563
 564 /**
 565  * show_scaling_available_governors - show the available CPUfreq governors
 566  */
 567 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 568                                                 char *buf)
 569 {
 570         ssize_t i = 0;
 571         struct cpufreq_governor *t;
 572
 573         if (!cpufreq_driver->target) {
 574                 i += sprintf(buf, "performance powersave");
 575                 goto out;
 576         }
 577
 578         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 579                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 580                     - (CPUFREQ_NAME_LEN + 2)))
 581                         goto out;
 582                 i += scnprintf(&buf[i], CPUFREQ_NAME_LEN, "%s ", t->name);
 583         }
 584 out:
 585         i += sprintf(&buf[i], "\n");
 586         return i;
 587 }
 588
 589 static ssize_t show_cpus(const struct cpumask *mask, char *buf)
 590 {
 591         ssize_t i = 0;
 592         unsigned int cpu;
 593
 594         for_each_cpu(cpu, mask) {
 595                 if (i)
 596                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 597                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 598                 if (i >= (PAGE_SIZE - 5))
 599                         break;
 600         }
 601         i += sprintf(&buf[i], "\n");
 602         return i;
 603 }
 604
 605 /**
 606  * show_related_cpus - show the CPUs affected by each transition even if
 607  * hw coordination is in use
 608  */
 609 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 610 {
 611         if (cpumask_empty(policy->related_cpus))
 612                 return show_cpus(policy->cpus, buf);
 613         return show_cpus(policy->related_cpus, buf);
 614 }
 615
 616 /**
 617  * show_affected_cpus - show the CPUs affected by each transition
 618  */
 619 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 620 {
 621         return show_cpus(policy->cpus, buf);
 622 }
 623
 624 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 625                                         const char *buf, size_t count)
 626 {
 627         unsigned int freq = 0;
 628         unsigned int ret;
 629
 630         if (!policy->governor || !policy->governor->store_setspeed)
 631                 return -EINVAL;
 632
 633         ret = sscanf(buf, "%u", &freq);
 634         if (ret != 1)
 635                 return -EINVAL;
 636
 637         policy->governor->store_setspeed(policy, freq);
 638
 639         return count;
 640 }
 641
 642 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 643 {
 644         if (!policy->governor || !policy->governor->show_setspeed)
 645                 return sprintf(buf, "<unsupported>\n");
 646
 647         return policy->governor->show_setspeed(policy, buf);
 648 }
 649
 650 #define define_one_ro(_name) \
 651 static struct freq_attr _name = \
 652 __ATTR(_name, 0444, show_##_name, NULL)
 653
 654 #define define_one_ro0400(_name) \
 655 static struct freq_attr _name = \
 656 __ATTR(_name, 0400, show_##_name, NULL)
 657
 658 #define define_one_rw(_name) \
 659 static struct freq_attr _name = \
 660 __ATTR(_name, 0644, show_##_name, store_##_name)
 661
 662 define_one_ro0400(cpuinfo_cur_freq);
 663 define_one_ro(cpuinfo_min_freq);
 664 define_one_ro(cpuinfo_max_freq);
 665 define_one_ro(cpuinfo_transition_latency);
 666 define_one_ro(scaling_available_governors);
 667 define_one_ro(scaling_driver);
 668 define_one_ro(scaling_cur_freq);
 669 define_one_ro(related_cpus);
 670 define_one_ro(affected_cpus);
 671 define_one_rw(scaling_min_freq);
 672 define_one_rw(scaling_max_freq);
 673 define_one_rw(scaling_governor);
 674 define_one_rw(scaling_setspeed);
 675
 676 static struct attribute *default_attrs[] = {
 677         &cpuinfo_min_freq.attr,
 678         &cpuinfo_max_freq.attr,
 679         &cpuinfo_transition_latency.attr,
 680         &scaling_min_freq.attr,
 681         &scaling_max_freq.attr,
 682         &affected_cpus.attr,
 683         &related_cpus.attr,
 684         &scaling_governor.attr,
 685         &scaling_driver.attr,
 686         &scaling_available_governors.attr,
 687         &scaling_setspeed.attr,
 688         NULL
 689 };
 690
 691 struct kobject *cpufreq_global_kobject;
 692 EXPORT_SYMBOL(cpufreq_global_kobject);
 693
 694 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 695 #define to_attr(a) container_of(a, struct freq_attr, attr)
 696
 697 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 698 {
 699         struct cpufreq_policy *policy = to_policy(kobj);
 700         struct freq_attr *fattr = to_attr(attr);
 701         ssize_t ret = -EINVAL;
 702         policy = cpufreq_cpu_get(policy->cpu);
 703         if (!policy)
 704                 goto no_policy;
 705
 706         if (lock_policy_rwsem_read(policy->cpu) < 0)
 707                 goto fail;
 708
 709         if (fattr->show)
 710                 ret = fattr->show(policy, buf);
 711         else
 712                 ret = -EIO;
 713
 714         unlock_policy_rwsem_read(policy->cpu);
 715 fail:
 716         cpufreq_cpu_put(policy);
 717 no_policy:
 718         return ret;
 719 }
 720
 721 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 722                      const char *buf, size_t count)
 723 {
 724         struct cpufreq_policy *policy = to_policy(kobj);
 725         struct freq_attr *fattr = to_attr(attr);
 726         ssize_t ret = -EINVAL;
 727         policy = cpufreq_cpu_get(policy->cpu);
 728         if (!policy)
 729                 goto no_policy;
 730
 731         if (lock_policy_rwsem_write(policy->cpu) < 0)
 732                 goto fail;
 733
 734         if (fattr->store)
 735                 ret = fattr->store(policy, buf, count);
 736         else
 737                 ret = -EIO;
 738
 739         unlock_policy_rwsem_write(policy->cpu);
 740 fail:
 741         cpufreq_cpu_put(policy);
 742 no_policy:
 743         return ret;
 744 }
 745
 746 static void cpufreq_sysfs_release(struct kobject *kobj)
 747 {
 748         struct cpufreq_policy *policy = to_policy(kobj);
 749         dprintk("last reference is dropped\n");
 750         complete(&policy->kobj_unregister);
 751 }
 752
 753 static struct sysfs_ops sysfs_ops = {
 754         .show   = show,
 755         .store  = store,
 756 };
 757
 758 static struct kobj_type ktype_cpufreq = {
 759         .sysfs_ops      = &sysfs_ops,
 760         .default_attrs  = default_attrs,
 761         .release        = cpufreq_sysfs_release,
 762 };
 763
 764 /*
 765  * Returns:
 766  *   Negative: Failure
 767  *   0:        Success
 768  *   Positive: When we have a managed CPU and the sysfs got symlinked
 769  */
 770 int cpufreq_add_dev_policy(unsigned int cpu, struct cpufreq_policy *policy,
 771                 struct sys_device *sys_dev)
 772 {
 773         int ret = 0;
 774 #ifdef CONFIG_SMP
 775         unsigned long flags;
 776         unsigned int j;
 777
 778 #ifdef CONFIG_HOTPLUG_CPU
 779         if (per_cpu(cpufreq_cpu_governor, cpu)) {
 780                 policy->governor = per_cpu(cpufreq_cpu_governor, cpu);
 781                 dprintk("Restoring governor %s for cpu %d\n",
 782                        policy->governor->name, cpu);
 783         }
 784 #endif
 785
 786         for_each_cpu(j, policy->cpus) {
 787                 struct cpufreq_policy *managed_policy;
 788
 789                 if (cpu == j)
 790                         continue;
 791
 792                 /* Check for existing affected CPUs.
 793                  * They may not be aware of it due to CPU Hotplug.
 794                  * cpufreq_cpu_put is called when the device is removed
 795                  * in __cpufreq_remove_dev()
 796                  */
 797                 managed_policy = cpufreq_cpu_get(j);
 798                 if (unlikely(managed_policy)) {
 799
 800                         /* Set proper policy_cpu */
 801                         unlock_policy_rwsem_write(cpu);
 802                         per_cpu(policy_cpu, cpu) = managed_policy->cpu;
 803
 804                         if (lock_policy_rwsem_write(cpu) < 0) {
 805                                 /* Should not go through policy unlock path */
 806                                 if (cpufreq_driver->exit)
 807                                         cpufreq_driver->exit(policy);
 808                                 cpufreq_cpu_put(managed_policy);
 809                                 return -EBUSY;
 810                         }
 811
 812                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 813                         cpumask_copy(managed_policy->cpus, policy->cpus);
 814                         per_cpu(cpufreq_cpu_data, cpu) = managed_policy;
 815                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 816
 817                         dprintk("CPU already managed, adding link\n");
 818                         ret = sysfs_create_link(&sys_dev->kobj,
 819                                                 &managed_policy->kobj,
 820                                                 "cpufreq");
 821                         if (ret)
 822                                 cpufreq_cpu_put(managed_policy);
 823                         /*
 824                          * Success. We only needed to be added to the mask.
 825                          * Call driver->exit() because only the cpu parent of
 826                          * the kobj needed to call init().
 827                          */
 828                         if (cpufreq_driver->exit)
 829                                 cpufreq_driver->exit(policy);
 830
 831                         if (!ret)
 832                                 return 1;
 833                         else
 834                                 return ret;
 835                 }
 836         }
 837 #endif
 838         return ret;
 839 }
 840
 841
 842 /* symlink affected CPUs */
 843 int cpufreq_add_dev_symlink(unsigned int cpu, struct cpufreq_policy *policy)
 844 {
 845         unsigned int j;
 846         int ret = 0;
 847
 848         for_each_cpu(j, policy->cpus) {
 849                 struct cpufreq_policy *managed_policy;
 850                 struct sys_device *cpu_sys_dev;
 851
 852                 if (j == cpu)
 853                         continue;
 854                 if (!cpu_online(j))
 855                         continue;
 856
 857                 dprintk("CPU %u already managed, adding link\n", j);
 858                 managed_policy = cpufreq_cpu_get(cpu);
 859                 cpu_sys_dev = get_cpu_sysdev(j);
 860                 ret = sysfs_create_link(&cpu_sys_dev->kobj, &policy->kobj,
 861                                         "cpufreq");
 862                 if (ret) {
 863                         cpufreq_cpu_put(managed_policy);
 864                         return ret;
 865                 }
 866         }
 867         return ret;
 868 }
 869
 870 int cpufreq_add_dev_interface(unsigned int cpu, struct cpufreq_policy *policy,
 871                 struct sys_device *sys_dev)
 872 {
 873         struct cpufreq_policy new_policy;
 874         struct freq_attr **drv_attr;
 875         unsigned long flags;
 876         int ret = 0;
 877         unsigned int j;
 878
 879         /* prepare interface data */
 880         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 881                                    &sys_dev->kobj, "cpufreq");
 882         if (ret)
 883                 return ret;
 884
 885         /* set up files for this cpu device */
 886         drv_attr = cpufreq_driver->attr;
 887         while ((drv_attr) && (*drv_attr)) {
 888                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 889                 if (ret)
 890                         goto err_out_kobj_put;
 891                 drv_attr++;
 892         }
 893         if (cpufreq_driver->get) {
 894                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 895                 if (ret)
 896                         goto err_out_kobj_put;
 897         }
 898         if (cpufreq_driver->target) {
 899                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 900                 if (ret)
 901                         goto err_out_kobj_put;
 902         }
 903
 904         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 905         for_each_cpu(j, policy->cpus) {
 906         if (!cpu_online(j))
 907                 continue;
 908                 per_cpu(cpufreq_cpu_data, j) = policy;
 909                 per_cpu(policy_cpu, j) = policy->cpu;
 910         }
 911         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 912
 913         ret = cpufreq_add_dev_symlink(cpu, policy);
 914         if (ret)
 915                 goto err_out_kobj_put;
 916
 917         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 918         /* assure that the starting sequence is run in __cpufreq_set_policy */
 919         policy->governor = NULL;
 920
 921         /* set default policy */
 922         ret = __cpufreq_set_policy(policy, &new_policy);
 923         policy->user_policy.policy = policy->policy;
 924         policy->user_policy.governor = policy->governor;
 925
 926         if (ret) {
 927                 dprintk("setting policy failed\n");
 928                 if (cpufreq_driver->exit)
 929                         cpufreq_driver->exit(policy);
 930         }
 931         return ret;
 932
 933 err_out_kobj_put:
 934         kobject_put(&policy->kobj);
 935         wait_for_completion(&policy->kobj_unregister);
 936         return ret;
 937 }
 938
 939
 940 /**
 941  * cpufreq_add_dev - add a CPU device
 942  *
 943  * Adds the cpufreq interface for a CPU device.
 944  *
 945  * The Oracle says: try running cpufreq registration/unregistration concurrently
 946  * with with cpu hotplugging and all hell will break loose. Tried to clean this
 947  * mess up, but more thorough testing is needed. - Mathieu
 948  */
 949 static int cpufreq_add_dev(struct sys_device *sys_dev)
 950 {
 951         unsigned int cpu = sys_dev->id;
 952         int ret = 0;
 953         struct cpufreq_policy *policy;
 954         unsigned long flags;
 955         unsigned int j;
 956
 957         if (cpu_is_offline(cpu))
 958                 return 0;
 959
 960         cpufreq_debug_disable_ratelimit();
 961         dprintk("adding CPU %u\n", cpu);
 962
 963 #ifdef CONFIG_SMP
 964         /* check whether a different CPU already registered this
 965          * CPU because it is in the same boat. */
 966         policy = cpufreq_cpu_get(cpu);
 967         if (unlikely(policy)) {
 968                 cpufreq_cpu_put(policy);
 969                 cpufreq_debug_enable_ratelimit();
 970                 return 0;
 971         }
 972 #endif
 973
 974         if (!try_module_get(cpufreq_driver->owner)) {
 975                 ret = -EINVAL;
 976                 goto module_out;
 977         }
 978
 979         ret = -ENOMEM;
 980         policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
 981         if (!policy)
 982                 goto nomem_out;
 983
 984         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
 985                 goto err_free_policy;
 986
 987         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
 988                 goto err_free_cpumask;
 989
 990         policy->cpu = cpu;
 991         cpumask_copy(policy->cpus, cpumask_of(cpu));
 992
 993         /* Initially set CPU itself as the policy_cpu */
 994         per_cpu(policy_cpu, cpu) = cpu;
 995         ret = (lock_policy_rwsem_write(cpu) < 0);
 996         WARN_ON(ret);
 997
 998         init_completion(&policy->kobj_unregister);
 999         INIT_WORK(&policy->update, handle_update);
1000
1001         /* Set governor before ->init, so that driver could check it */
1002         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
1003         /* call driver. From then on the cpufreq must be able
1004          * to accept all calls to ->verify and ->setpolicy for this CPU
1005          */
1006         ret = cpufreq_driver->init(policy);
1007         if (ret) {
1008                 dprintk("initialization failed\n");
1009                 goto err_unlock_policy;
1010         }
1011         policy->user_policy.min = policy->min;
1012         policy->user_policy.max = policy->max;
1013
1014         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1015                                      CPUFREQ_START, policy);
1016
1017         ret = cpufreq_add_dev_policy(cpu, policy, sys_dev);
1018         if (ret) {
1019                 if (ret > 0)
1020                         /* This is a managed cpu, symlink created,
1021                            exit with 0 */
1022                         ret = 0;
1023                 goto err_unlock_policy;
1024         }
1025
1026         ret = cpufreq_add_dev_interface(cpu, policy, sys_dev);
1027         if (ret)
1028                 goto err_out_unregister;
1029
1030         unlock_policy_rwsem_write(cpu);
1031
1032         kobject_uevent(&policy->kobj, KOBJ_ADD);
1033         module_put(cpufreq_driver->owner);
1034         dprintk("initialization complete\n");
1035         cpufreq_debug_enable_ratelimit();
1036
1037         return 0;
1038
1039
1040 err_out_unregister:
1041         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1042         for_each_cpu(j, policy->cpus)
1043                 per_cpu(cpufreq_cpu_data, j) = NULL;
1044         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1045
1046         kobject_put(&policy->kobj);
1047         wait_for_completion(&policy->kobj_unregister);
1048
1049 err_unlock_policy:
1050         unlock_policy_rwsem_write(cpu);
1051 err_free_cpumask:
1052         free_cpumask_var(policy->cpus);
1053 err_free_policy:
1054         kfree(policy);
1055 nomem_out:
1056         module_put(cpufreq_driver->owner);
1057 module_out:
1058         cpufreq_debug_enable_ratelimit();
1059         return ret;
1060 }
1061
1062
1063 /**
1064  * __cpufreq_remove_dev - remove a CPU device
1065  *
1066  * Removes the cpufreq interface for a CPU device.
1067  * Caller should already have policy_rwsem in write mode for this CPU.
1068  * This routine frees the rwsem before returning.
1069  */
1070 static int __cpufreq_remove_dev(struct sys_device *sys_dev)
1071 {
1072         unsigned int cpu = sys_dev->id;
1073         unsigned long flags;
1074         struct cpufreq_policy *data;
1075 #ifdef CONFIG_SMP
1076         struct sys_device *cpu_sys_dev;
1077         unsigned int j;
1078 #endif
1079
1080         cpufreq_debug_disable_ratelimit();
1081         dprintk("unregistering CPU %u\n", cpu);
1082
1083         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1084         data = per_cpu(cpufreq_cpu_data, cpu);
1085
1086         if (!data) {
1087                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1088                 cpufreq_debug_enable_ratelimit();
1089                 unlock_policy_rwsem_write(cpu);
1090                 return -EINVAL;
1091         }
1092         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1093
1094
1095 #ifdef CONFIG_SMP
1096         /* if this isn't the CPU which is the parent of the kobj, we
1097          * only need to unlink, put and exit
1098          */
1099         if (unlikely(cpu != data->cpu)) {
1100                 dprintk("removing link\n");
1101                 cpumask_clear_cpu(cpu, data->cpus);
1102                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1103                 sysfs_remove_link(&sys_dev->kobj, "cpufreq");
1104                 cpufreq_cpu_put(data);
1105                 cpufreq_debug_enable_ratelimit();
1106                 unlock_policy_rwsem_write(cpu);
1107                 return 0;
1108         }
1109 #endif
1110
1111 #ifdef CONFIG_SMP
1112
1113 #ifdef CONFIG_HOTPLUG_CPU
1114         per_cpu(cpufreq_cpu_governor, cpu) = data->governor;
1115 #endif
1116
1117         /* if we have other CPUs still registered, we need to unlink them,
1118          * or else wait_for_completion below will lock up. Clean the
1119          * per_cpu(cpufreq_cpu_data) while holding the lock, and remove
1120          * the sysfs links afterwards.
1121          */
1122         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1123                 for_each_cpu(j, data->cpus) {
1124                         if (j == cpu)
1125                                 continue;
1126                         per_cpu(cpufreq_cpu_data, j) = NULL;
1127                 }
1128         }
1129
1130         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1131
1132         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1133                 for_each_cpu(j, data->cpus) {
1134                         if (j == cpu)
1135                                 continue;
1136                         dprintk("removing link for cpu %u\n", j);
1137 #ifdef CONFIG_HOTPLUG_CPU
1138                         per_cpu(cpufreq_cpu_governor, j) = data->governor;
1139 #endif
1140                         cpu_sys_dev = get_cpu_sysdev(j);
1141                         sysfs_remove_link(&cpu_sys_dev->kobj, "cpufreq");
1142                         cpufreq_cpu_put(data);
1143                 }
1144         }
1145 #else
1146         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1147 #endif
1148
1149         if (cpufreq_driver->target)
1150                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1151
1152         kobject_put(&data->kobj);
1153
1154         /* we need to make sure that the underlying kobj is actually
1155          * not referenced anymore by anybody before we proceed with
1156          * unloading.
1157          */
1158         dprintk("waiting for dropping of refcount\n");
1159         wait_for_completion(&data->kobj_unregister);
1160         dprintk("wait complete\n");
1161
1162         if (cpufreq_driver->exit)
1163                 cpufreq_driver->exit(data);
1164
1165         unlock_policy_rwsem_write(cpu);
1166
1167         free_cpumask_var(data->related_cpus);
1168         free_cpumask_var(data->cpus);
1169         kfree(data);
1170         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1171
1172         cpufreq_debug_enable_ratelimit();
1173         return 0;
1174 }
1175
1176
1177 static int cpufreq_remove_dev(struct sys_device *sys_dev)
1178 {
1179         unsigned int cpu = sys_dev->id;
1180         int retval;
1181
1182         if (cpu_is_offline(cpu))
1183                 return 0;
1184
1185         if (unlikely(lock_policy_rwsem_write(cpu)))
1186                 BUG();
1187
1188         retval = __cpufreq_remove_dev(sys_dev);
1189         return retval;
1190 }
1191
1192
1193 static void handle_update(struct work_struct *work)
1194 {
1195         struct cpufreq_policy *policy =
1196                 container_of(work, struct cpufreq_policy, update);
1197         unsigned int cpu = policy->cpu;
1198         dprintk("handle_update for cpu %u called\n", cpu);
1199         cpufreq_update_policy(cpu);
1200 }
1201
1202 /**
1203  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1204  *      @cpu: cpu number
1205  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1206  *      @new_freq: CPU frequency the CPU actually runs at
1207  *
1208  *      We adjust to current frequency first, and need to clean up later.
1209  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1210  */
1211 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1212                                 unsigned int new_freq)
1213 {
1214         struct cpufreq_freqs freqs;
1215
1216         dprintk("Warning: CPU frequency out of sync: cpufreq and timing "
1217                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1218
1219         freqs.cpu = cpu;
1220         freqs.old = old_freq;
1221         freqs.new = new_freq;
1222         cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
1223         cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
1224 }
1225
1226
1227 /**
1228  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1229  * @cpu: CPU number
1230  *
1231  * This is the last known freq, without actually getting it from the driver.
1232  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1233  */
1234 unsigned int cpufreq_quick_get(unsigned int cpu)
1235 {
1236         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1237         unsigned int ret_freq = 0;
1238
1239         if (policy) {
1240                 ret_freq = policy->cur;
1241                 cpufreq_cpu_put(policy);
1242         }
1243
1244         return ret_freq;
1245 }
1246 EXPORT_SYMBOL(cpufreq_quick_get);
1247
1248
1249 static unsigned int __cpufreq_get(unsigned int cpu)
1250 {
1251         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1252         unsigned int ret_freq = 0;
1253
1254         if (!cpufreq_driver->get)
1255                 return ret_freq;
1256
1257         ret_freq = cpufreq_driver->get(cpu);
1258
1259         if (ret_freq && policy->cur &&
1260                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1261                 /* verify no discrepancy between actual and
1262                                         saved value exists */
1263                 if (unlikely(ret_freq != policy->cur)) {
1264                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1265                         schedule_work(&policy->update);
1266                 }
1267         }
1268
1269         return ret_freq;
1270 }
1271
1272 /**
1273  * cpufreq_get - get the current CPU frequency (in kHz)
1274  * @cpu: CPU number
1275  *
1276  * Get the CPU current (static) CPU frequency
1277  */
1278 unsigned int cpufreq_get(unsigned int cpu)
1279 {
1280         unsigned int ret_freq = 0;
1281         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1282
1283         if (!policy)
1284                 goto out;
1285
1286         if (unlikely(lock_policy_rwsem_read(cpu)))
1287                 goto out_policy;
1288
1289         ret_freq = __cpufreq_get(cpu);
1290
1291         unlock_policy_rwsem_read(cpu);
1292
1293 out_policy:
1294         cpufreq_cpu_put(policy);
1295 out:
1296         return ret_freq;
1297 }
1298 EXPORT_SYMBOL(cpufreq_get);
1299
1300
1301 /**
1302  *      cpufreq_suspend - let the low level driver prepare for suspend
1303  */
1304
1305 static int cpufreq_suspend(struct sys_device *sysdev, pm_message_t pmsg)
1306 {
1307         int ret = 0;
1308
1309         int cpu = sysdev->id;
1310         struct cpufreq_policy *cpu_policy;
1311
1312         dprintk("suspending cpu %u\n", cpu);
1313
1314         if (!cpu_online(cpu))
1315                 return 0;
1316
1317         /* we may be lax here as interrupts are off. Nonetheless
1318          * we need to grab the correct cpu policy, as to check
1319          * whether we really run on this CPU.
1320          */
1321
1322         cpu_policy = cpufreq_cpu_get(cpu);
1323         if (!cpu_policy)
1324                 return -EINVAL;
1325
1326         /* only handle each CPU group once */
1327         if (unlikely(cpu_policy->cpu != cpu))
1328                 goto out;
1329
1330         if (cpufreq_driver->suspend) {
1331                 ret = cpufreq_driver->suspend(cpu_policy, pmsg);
1332                 if (ret)
1333                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1334                                         "step on CPU %u\n", cpu_policy->cpu);
1335         }
1336
1337 out:
1338         cpufreq_cpu_put(cpu_policy);
1339         return ret;
1340 }
1341
1342 /**
1343  *      cpufreq_resume -  restore proper CPU frequency handling after resume
1344  *
1345  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1346  *      2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1347  *          restored. It will verify that the current freq is in sync with
1348  *          what we believe it to be. This is a bit later than when it
1349  *          should be, but nonethteless it's better than calling
1350  *          cpufreq_driver->get() here which might re-enable interrupts...
1351  */
1352 static int cpufreq_resume(struct sys_device *sysdev)
1353 {
1354         int ret = 0;
1355
1356         int cpu = sysdev->id;
1357         struct cpufreq_policy *cpu_policy;
1358
1359         dprintk("resuming cpu %u\n", cpu);
1360
1361         if (!cpu_online(cpu))
1362                 return 0;
1363
1364         /* we may be lax here as interrupts are off. Nonetheless
1365          * we need to grab the correct cpu policy, as to check
1366          * whether we really run on this CPU.
1367          */
1368
1369         cpu_policy = cpufreq_cpu_get(cpu);
1370         if (!cpu_policy)
1371                 return -EINVAL;
1372
1373         /* only handle each CPU group once */
1374         if (unlikely(cpu_policy->cpu != cpu))
1375                 goto fail;
1376
1377         if (cpufreq_driver->resume) {
1378                 ret = cpufreq_driver->resume(cpu_policy);
1379                 if (ret) {
1380                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1381                                         "step on CPU %u\n", cpu_policy->cpu);
1382                         goto fail;
1383                 }
1384         }
1385
1386         schedule_work(&cpu_policy->update);
1387
1388 fail:
1389         cpufreq_cpu_put(cpu_policy);
1390         return ret;
1391 }
1392
1393 static struct sysdev_driver cpufreq_sysdev_driver = {
1394         .add            = cpufreq_add_dev,
1395         .remove         = cpufreq_remove_dev,
1396         .suspend        = cpufreq_suspend,
1397         .resume         = cpufreq_resume,
1398 };
1399
1400
1401 /*********************************************************************
1402  *                     NOTIFIER LISTS INTERFACE                      *
1403  *********************************************************************/
1404
1405 /**
1406  *      cpufreq_register_notifier - register a driver with cpufreq
1407  *      @nb: notifier function to register
1408  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1409  *
1410  *      Add a driver to one of two lists: either a list of drivers that
1411  *      are notified about clock rate changes (once before and once after
1412  *      the transition), or a list of drivers that are notified about
1413  *      changes in cpufreq policy.
1414  *
1415  *      This function may sleep, and has the same return conditions as
1416  *      blocking_notifier_chain_register.
1417  */
1418 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1419 {
1420         int ret;
1421
1422         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1423
1424         switch (list) {
1425         case CPUFREQ_TRANSITION_NOTIFIER:
1426                 ret = srcu_notifier_chain_register(
1427                                 &cpufreq_transition_notifier_list, nb);
1428                 break;
1429         case CPUFREQ_POLICY_NOTIFIER:
1430                 ret = blocking_notifier_chain_register(
1431                                 &cpufreq_policy_notifier_list, nb);
1432                 break;
1433         default:
1434                 ret = -EINVAL;
1435         }
1436
1437         return ret;
1438 }
1439 EXPORT_SYMBOL(cpufreq_register_notifier);
1440
1441
1442 /**
1443  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1444  *      @nb: notifier block to be unregistered
1445  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1446  *
1447  *      Remove a driver from the CPU frequency notifier list.
1448  *
1449  *      This function may sleep, and has the same return conditions as
1450  *      blocking_notifier_chain_unregister.
1451  */
1452 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1453 {
1454         int ret;
1455
1456         switch (list) {
1457         case CPUFREQ_TRANSITION_NOTIFIER:
1458                 ret = srcu_notifier_chain_unregister(
1459                                 &cpufreq_transition_notifier_list, nb);
1460                 break;
1461         case CPUFREQ_POLICY_NOTIFIER:
1462                 ret = blocking_notifier_chain_unregister(
1463                                 &cpufreq_policy_notifier_list, nb);
1464                 break;
1465         default:
1466                 ret = -EINVAL;
1467         }
1468
1469         return ret;
1470 }
1471 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1472
1473
1474 /*********************************************************************
1475  *                              GOVERNORS                            *
1476  *********************************************************************/
1477
1478
1479 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1480                             unsigned int target_freq,
1481                             unsigned int relation)
1482 {
1483         int retval = -EINVAL;
1484
1485         dprintk("target for CPU %u: %u kHz, relation %u\n", policy->cpu,
1486                 target_freq, relation);
1487         if (cpu_online(policy->cpu) && cpufreq_driver->target)
1488                 retval = cpufreq_driver->target(policy, target_freq, relation);
1489
1490         return retval;
1491 }
1492 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1493
1494 int cpufreq_driver_target(struct cpufreq_policy *policy,
1495                           unsigned int target_freq,
1496                           unsigned int relation)
1497 {
1498         int ret = -EINVAL;
1499
1500         policy = cpufreq_cpu_get(policy->cpu);
1501         if (!policy)
1502                 goto no_policy;
1503
1504         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1505                 goto fail;
1506
1507         ret = __cpufreq_driver_target(policy, target_freq, relation);
1508
1509         unlock_policy_rwsem_write(policy->cpu);
1510
1511 fail:
1512         cpufreq_cpu_put(policy);
1513 no_policy:
1514         return ret;
1515 }
1516 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1517
1518 int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1519 {
1520         int ret = 0;
1521
1522         policy = cpufreq_cpu_get(policy->cpu);
1523         if (!policy)
1524                 return -EINVAL;
1525
1526         if (cpu_online(cpu) && cpufreq_driver->getavg)
1527                 ret = cpufreq_driver->getavg(policy, cpu);
1528
1529         cpufreq_cpu_put(policy);
1530         return ret;
1531 }
1532 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1533
1534 /*
1535  * when "event" is CPUFREQ_GOV_LIMITS
1536  */
1537
1538 static int __cpufreq_governor(struct cpufreq_policy *policy,
1539                                         unsigned int event)
1540 {
1541         int ret;
1542
1543         /* Only must be defined when default governor is known to have latency
1544            restrictions, like e.g. conservative or ondemand.
1545            That this is the case is already ensured in Kconfig
1546         */
1547 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1548         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1549 #else
1550         struct cpufreq_governor *gov = NULL;
1551 #endif
1552
1553         if (policy->governor->max_transition_latency &&
1554             policy->cpuinfo.transition_latency >
1555             policy->governor->max_transition_latency) {
1556                 if (!gov)
1557                         return -EINVAL;
1558                 else {
1559                         printk(KERN_WARNING "%s governor failed, too long"
1560                                " transition latency of HW, fallback"
1561                                " to %s governor\n",
1562                                policy->governor->name,
1563                                gov->name);
1564                         policy->governor = gov;
1565                 }
1566         }
1567
1568         if (!try_module_get(policy->governor->owner))
1569                 return -EINVAL;
1570
1571         dprintk("__cpufreq_governor for CPU %u, event %u\n",
1572                                                 policy->cpu, event);
1573         ret = policy->governor->governor(policy, event);
1574
1575         /* we keep one module reference alive for
1576                         each CPU governed by this CPU */
1577         if ((event != CPUFREQ_GOV_START) || ret)
1578                 module_put(policy->governor->owner);
1579         if ((event == CPUFREQ_GOV_STOP) && !ret)
1580                 module_put(policy->governor->owner);
1581
1582         return ret;
1583 }
1584
1585
1586 int cpufreq_register_governor(struct cpufreq_governor *governor)
1587 {
1588         int err;
1589
1590         if (!governor)
1591                 return -EINVAL;
1592
1593         mutex_lock(&cpufreq_governor_mutex);
1594
1595         err = -EBUSY;
1596         if (__find_governor(governor->name) == NULL) {
1597                 err = 0;
1598                 list_add(&governor->governor_list, &cpufreq_governor_list);
1599         }
1600
1601         mutex_unlock(&cpufreq_governor_mutex);
1602         return err;
1603 }
1604 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1605
1606
1607 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1608 {
1609         if (!governor)
1610                 return;
1611
1612         mutex_lock(&cpufreq_governor_mutex);
1613         list_del(&governor->governor_list);
1614         mutex_unlock(&cpufreq_governor_mutex);
1615         return;
1616 }
1617 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1618
1619
1620
1621 /*********************************************************************
1622  *                          POLICY INTERFACE                         *
1623  *********************************************************************/
1624
1625 /**
1626  * cpufreq_get_policy - get the current cpufreq_policy
1627  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1628  *      is written
1629  *
1630  * Reads the current cpufreq policy.
1631  */
1632 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1633 {
1634         struct cpufreq_policy *cpu_policy;
1635         if (!policy)
1636                 return -EINVAL;
1637
1638         cpu_policy = cpufreq_cpu_get(cpu);
1639         if (!cpu_policy)
1640                 return -EINVAL;
1641
1642         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1643
1644         cpufreq_cpu_put(cpu_policy);
1645         return 0;
1646 }
1647 EXPORT_SYMBOL(cpufreq_get_policy);
1648
1649
1650 /*
1651  * data   : current policy.
1652  * policy : policy to be set.
1653  */
1654 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1655                                 struct cpufreq_policy *policy)
1656 {
1657         int ret = 0;
1658
1659         cpufreq_debug_disable_ratelimit();
1660         dprintk("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1661                 policy->min, policy->max);
1662
1663         memcpy(&policy->cpuinfo, &data->cpuinfo,
1664                                 sizeof(struct cpufreq_cpuinfo));
1665
1666         if (policy->min > data->max || policy->max < data->min) {
1667                 ret = -EINVAL;
1668                 goto error_out;
1669         }
1670
1671         /* verify the cpu speed can be set within this limit */
1672         ret = cpufreq_driver->verify(policy);
1673         if (ret)
1674                 goto error_out;
1675
1676         /* adjust if necessary - all reasons */
1677         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1678                         CPUFREQ_ADJUST, policy);
1679
1680         /* adjust if necessary - hardware incompatibility*/
1681         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1682                         CPUFREQ_INCOMPATIBLE, policy);
1683
1684         /* verify the cpu speed can be set within this limit,
1685            which might be different to the first one */
1686         ret = cpufreq_driver->verify(policy);
1687         if (ret)
1688                 goto error_out;
1689
1690         /* notification of the new policy */
1691         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1692                         CPUFREQ_NOTIFY, policy);
1693
1694         data->min = policy->min;
1695         data->max = policy->max;
1696
1697         dprintk("new min and max freqs are %u - %u kHz\n",
1698                                         data->min, data->max);
1699
1700         if (cpufreq_driver->setpolicy) {
1701                 data->policy = policy->policy;
1702                 dprintk("setting range\n");
1703                 ret = cpufreq_driver->setpolicy(policy);
1704         } else {
1705                 if (policy->governor != data->governor) {
1706                         /* save old, working values */
1707                         struct cpufreq_governor *old_gov = data->governor;
1708
1709                         dprintk("governor switch\n");
1710
1711                         /* end old governor */
1712                         if (data->governor) {
1713                                 /*
1714                                  * Need to release the rwsem around governor
1715                                  * stop due to lock dependency between
1716                                  * cancel_delayed_work_sync and the read lock
1717                                  * taken in the delayed work handler.
1718                                  */
1719                                 unlock_policy_rwsem_write(data->cpu);
1720                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1721                                 lock_policy_rwsem_write(data->cpu);
1722                         }
1723
1724                         /* start new governor */
1725                         data->governor = policy->governor;
1726                         if (__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1727                                 /* new governor failed, so re-start old one */
1728                                 dprintk("starting governor %s failed\n",
1729                                                         data->governor->name);
1730                                 if (old_gov) {
1731                                         data->governor = old_gov;
1732                                         __cpufreq_governor(data,
1733                                                            CPUFREQ_GOV_START);
1734                                 }
1735                                 ret = -EINVAL;
1736                                 goto error_out;
1737                         }
1738                         /* might be a policy change, too, so fall through */
1739                 }
1740                 dprintk("governor: change or update limits\n");
1741                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1742         }
1743
1744 error_out:
1745         cpufreq_debug_enable_ratelimit();
1746         return ret;
1747 }
1748
1749 /**
1750  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1751  *      @cpu: CPU which shall be re-evaluated
1752  *
1753  *      Usefull for policy notifiers which have different necessities
1754  *      at different times.
1755  */
1756 int cpufreq_update_policy(unsigned int cpu)
1757 {
1758         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1759         struct cpufreq_policy policy;
1760         int ret;
1761
1762         if (!data) {
1763                 ret = -ENODEV;
1764                 goto no_policy;
1765         }
1766
1767         if (unlikely(lock_policy_rwsem_write(cpu))) {
1768                 ret = -EINVAL;
1769                 goto fail;
1770         }
1771
1772         dprintk("updating policy for CPU %u\n", cpu);
1773         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1774         policy.min = data->user_policy.min;
1775         policy.max = data->user_policy.max;
1776         policy.policy = data->user_policy.policy;
1777         policy.governor = data->user_policy.governor;
1778
1779         /* BIOS might change freq behind our back
1780           -> ask driver for current freq and notify governors about a change */
1781         if (cpufreq_driver->get) {
1782                 policy.cur = cpufreq_driver->get(cpu);
1783                 if (!data->cur) {
1784                         dprintk("Driver did not initialize current freq");
1785                         data->cur = policy.cur;
1786                 } else {
1787                         if (data->cur != policy.cur)
1788                                 cpufreq_out_of_sync(cpu, data->cur,
1789                                                                 policy.cur);
1790                 }
1791         }
1792
1793         ret = __cpufreq_set_policy(data, &policy);
1794
1795         unlock_policy_rwsem_write(cpu);
1796
1797 fail:
1798         cpufreq_cpu_put(data);
1799 no_policy:
1800         return ret;
1801 }
1802 EXPORT_SYMBOL(cpufreq_update_policy);
1803
1804 static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1805                                         unsigned long action, void *hcpu)
1806 {
1807         unsigned int cpu = (unsigned long)hcpu;
1808         struct sys_device *sys_dev;
1809
1810         sys_dev = get_cpu_sysdev(cpu);
1811         if (sys_dev) {
1812                 switch (action) {
1813                 case CPU_ONLINE:
1814                 case CPU_ONLINE_FROZEN:
1815                         cpufreq_add_dev(sys_dev);
1816                         break;
1817                 case CPU_DOWN_PREPARE:
1818                 case CPU_DOWN_PREPARE_FROZEN:
1819                         if (unlikely(lock_policy_rwsem_write(cpu)))
1820                                 BUG();
1821
1822                         __cpufreq_remove_dev(sys_dev);
1823                         break;
1824                 case CPU_DOWN_FAILED:
1825                 case CPU_DOWN_FAILED_FROZEN:
1826                         cpufreq_add_dev(sys_dev);
1827                         break;
1828                 }
1829         }
1830         return NOTIFY_OK;
1831 }
1832
1833 static struct notifier_block __refdata cpufreq_cpu_notifier =
1834 {
1835     .notifier_call = cpufreq_cpu_callback,
1836 };
1837
1838 /*********************************************************************
1839  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1840  *********************************************************************/
1841
1842 /**
1843  * cpufreq_register_driver - register a CPU Frequency driver
1844  * @driver_data: A struct cpufreq_driver containing the values#
1845  * submitted by the CPU Frequency driver.
1846  *
1847  *   Registers a CPU Frequency driver to this core code. This code
1848  * returns zero on success, -EBUSY when another driver got here first
1849  * (and isn't unregistered in the meantime).
1850  *
1851  */
1852 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1853 {
1854         unsigned long flags;
1855         int ret;
1856
1857         if (!driver_data || !driver_data->verify || !driver_data->init ||
1858             ((!driver_data->setpolicy) && (!driver_data->target)))
1859                 return -EINVAL;
1860
1861         dprintk("trying to register driver %s\n", driver_data->name);
1862
1863         if (driver_data->setpolicy)
1864                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1865
1866         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1867         if (cpufreq_driver) {
1868                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1869                 return -EBUSY;
1870         }
1871         cpufreq_driver = driver_data;
1872         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1873
1874         ret = sysdev_driver_register(&cpu_sysdev_class,
1875                                         &cpufreq_sysdev_driver);
1876
1877         if ((!ret) && !(cpufreq_driver->flags & CPUFREQ_STICKY)) {
1878                 int i;
1879                 ret = -ENODEV;
1880
1881                 /* check for at least one working CPU */
1882                 for (i = 0; i < nr_cpu_ids; i++)
1883                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
1884                                 ret = 0;
1885                                 break;
1886                         }
1887
1888                 /* if all ->init() calls failed, unregister */
1889                 if (ret) {
1890                         dprintk("no CPU initialized for driver %s\n",
1891                                                         driver_data->name);
1892                         sysdev_driver_unregister(&cpu_sysdev_class,
1893                                                 &cpufreq_sysdev_driver);
1894
1895                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1896                         cpufreq_driver = NULL;
1897                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1898                 }
1899         }
1900
1901         if (!ret) {
1902                 register_hotcpu_notifier(&cpufreq_cpu_notifier);
1903                 dprintk("driver %s up and running\n", driver_data->name);
1904                 cpufreq_debug_enable_ratelimit();
1905         }
1906
1907         return ret;
1908 }
1909 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
1910
1911
1912 /**
1913  * cpufreq_unregister_driver - unregister the current CPUFreq driver
1914  *
1915  *    Unregister the current CPUFreq driver. Only call this if you have
1916  * the right to do so, i.e. if you have succeeded in initialising before!
1917  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
1918  * currently not initialised.
1919  */
1920 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
1921 {
1922         unsigned long flags;
1923
1924         cpufreq_debug_disable_ratelimit();
1925
1926         if (!cpufreq_driver || (driver != cpufreq_driver)) {
1927                 cpufreq_debug_enable_ratelimit();
1928                 return -EINVAL;
1929         }
1930
1931         dprintk("unregistering driver %s\n", driver->name);
1932
1933         sysdev_driver_unregister(&cpu_sysdev_class, &cpufreq_sysdev_driver);
1934         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
1935
1936         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1937         cpufreq_driver = NULL;
1938         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1939
1940         return 0;
1941 }
1942 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
1943
1944 static int __init cpufreq_core_init(void)
1945 {
1946         int cpu;
1947
1948         for_each_possible_cpu(cpu) {
1949                 per_cpu(policy_cpu, cpu) = -1;
1950                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
1951         }
1952
1953         cpufreq_global_kobject = kobject_create_and_add("cpufreq",
1954                                                 &cpu_sysdev_class.kset.kobj);
1955         BUG_ON(!cpufreq_global_kobject);
1956
1957         return 0;
1958 }
1959 core_initcall(cpufreq_core_init);