drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *
   7  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   8  *      Added handling for CPU hotplug
   9  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  10  *      Fix handling for CPU hotplug -- affected CPUs
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License version 2 as
  14  * published by the Free Software Foundation.
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <asm/cputime.h>
  21 #include <linux/kernel.h>
  22 #include <linux/kernel_stat.h>
  23 #include <linux/module.h>
  24 #include <linux/init.h>
  25 #include <linux/notifier.h>
  26 #include <linux/cpufreq.h>
  27 #include <linux/delay.h>
  28 #include <linux/interrupt.h>
  29 #include <linux/spinlock.h>
  30 #include <linux/tick.h>
  31 #include <linux/device.h>
  32 #include <linux/slab.h>
  33 #include <linux/cpu.h>
  34 #include <linux/completion.h>
  35 #include <linux/mutex.h>
  36 #include <linux/syscore_ops.h>
  37
  38 #include <trace/events/power.h>
  39
  40 /**
  41  * The "cpufreq driver" - the arch- or hardware-dependent low
  42  * level driver of CPUFreq support, and its spinlock. This lock
  43  * also protects the cpufreq_cpu_data array.
  44  */
  45 static struct cpufreq_driver *cpufreq_driver;
  46 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  47 #ifdef CONFIG_HOTPLUG_CPU
  48 /* This one keeps track of the previously set governor of a removed CPU */
  49 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  50 #endif
  51 static DEFINE_RWLOCK(cpufreq_driver_lock);
  52
  53 /*
  54  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  55  * all cpufreq/hotplug/workqueue/etc related lock issues.
  56  *
  57  * The rules for this semaphore:
  58  * - Any routine that wants to read from the policy structure will
  59  *   do a down_read on this semaphore.
  60  * - Any routine that will write to the policy structure and/or may take away
  61  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  62  *   mode before doing so.
  63  *
  64  * Additional rules:
  65  * - Governor routines that can be called in cpufreq hotplug path should not
  66  *   take this sem as top level hotplug notifier handler takes this.
  67  * - Lock should not be held across
  68  *     __cpufreq_governor(data, CPUFREQ_GOV_STOP);
  69  */
  70 static DEFINE_PER_CPU(int, cpufreq_policy_cpu);
  71 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  72
  73 #define lock_policy_rwsem(mode, cpu)                                    \
  74 static int lock_policy_rwsem_##mode(int cpu)                            \
  75 {                                                                       \
  76         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);              \
  77         BUG_ON(policy_cpu == -1);                                       \
  78         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  79                                                                         \
  80         return 0;                                                       \
  81 }
  82
  83 lock_policy_rwsem(read, cpu);
  84 lock_policy_rwsem(write, cpu);
  85
  86 #define unlock_policy_rwsem(mode, cpu)                                  \
  87 static void unlock_policy_rwsem_##mode(int cpu)                         \
  88 {                                                                       \
  89         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);              \
  90         BUG_ON(policy_cpu == -1);                                       \
  91         up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));              \
  92 }
  93
  94 unlock_policy_rwsem(read, cpu);
  95 unlock_policy_rwsem(write, cpu);
  96
  97 /* internal prototypes */
  98 static int __cpufreq_governor(struct cpufreq_policy *policy,
  99                 unsigned int event);
 100 static unsigned int __cpufreq_get(unsigned int cpu);
 101 static void handle_update(struct work_struct *work);
 102
 103 /**
 104  * Two notifier lists: the "policy" list is involved in the
 105  * validation process for a new CPU frequency policy; the
 106  * "transition" list for kernel code that needs to handle
 107  * changes to devices when the CPU clock speed changes.
 108  * The mutex locks both lists.
 109  */
 110 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 111 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 112
 113 static bool init_cpufreq_transition_notifier_list_called;
 114 static int __init init_cpufreq_transition_notifier_list(void)
 115 {
 116         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 117         init_cpufreq_transition_notifier_list_called = true;
 118         return 0;
 119 }
 120 pure_initcall(init_cpufreq_transition_notifier_list);
 121
 122 static int off __read_mostly;
 123 static int cpufreq_disabled(void)
 124 {
 125         return off;
 126 }
 127 void disable_cpufreq(void)
 128 {
 129         off = 1;
 130 }
 131 static LIST_HEAD(cpufreq_governor_list);
 132 static DEFINE_MUTEX(cpufreq_governor_mutex);
 133
 134 bool have_governor_per_policy(void)
 135 {
 136         return cpufreq_driver->have_governor_per_policy;
 137 }
 138 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 139
 140 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 141 {
 142         if (have_governor_per_policy())
 143                 return &policy->kobj;
 144         else
 145                 return cpufreq_global_kobject;
 146 }
 147 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 148
 149 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 150 {
 151         u64 idle_time;
 152         u64 cur_wall_time;
 153         u64 busy_time;
 154
 155         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 156
 157         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 158         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 159         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 160         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 161         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 162         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 163
 164         idle_time = cur_wall_time - busy_time;
 165         if (wall)
 166                 *wall = cputime_to_usecs(cur_wall_time);
 167
 168         return cputime_to_usecs(idle_time);
 169 }
 170
 171 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 172 {
 173         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 174
 175         if (idle_time == -1ULL)
 176                 return get_cpu_idle_time_jiffy(cpu, wall);
 177         else if (!io_busy)
 178                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 179
 180         return idle_time;
 181 }
 182 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 183
 184 static struct cpufreq_policy *__cpufreq_cpu_get(unsigned int cpu, bool sysfs)
 185 {
 186         struct cpufreq_policy *data;
 187         unsigned long flags;
 188
 189         if (cpu >= nr_cpu_ids)
 190                 goto err_out;
 191
 192         /* get the cpufreq driver */
 193         read_lock_irqsave(&cpufreq_driver_lock, flags);
 194
 195         if (!cpufreq_driver)
 196                 goto err_out_unlock;
 197
 198         if (!try_module_get(cpufreq_driver->owner))
 199                 goto err_out_unlock;
 200
 201
 202         /* get the CPU */
 203         data = per_cpu(cpufreq_cpu_data, cpu);
 204
 205         if (!data)
 206                 goto err_out_put_module;
 207
 208         if (!sysfs && !kobject_get(&data->kobj))
 209                 goto err_out_put_module;
 210
 211         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 212         return data;
 213
 214 err_out_put_module:
 215         module_put(cpufreq_driver->owner);
 216 err_out_unlock:
 217         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 218 err_out:
 219         return NULL;
 220 }
 221
 222 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 223 {
 224         if (cpufreq_disabled())
 225                 return NULL;
 226
 227         return __cpufreq_cpu_get(cpu, false);
 228 }
 229 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 230
 231 static struct cpufreq_policy *cpufreq_cpu_get_sysfs(unsigned int cpu)
 232 {
 233         return __cpufreq_cpu_get(cpu, true);
 234 }
 235
 236 static void __cpufreq_cpu_put(struct cpufreq_policy *data, bool sysfs)
 237 {
 238         if (!sysfs)
 239                 kobject_put(&data->kobj);
 240         module_put(cpufreq_driver->owner);
 241 }
 242
 243 void cpufreq_cpu_put(struct cpufreq_policy *data)
 244 {
 245         if (cpufreq_disabled())
 246                 return;
 247
 248         __cpufreq_cpu_put(data, false);
 249 }
 250 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 251
 252 static void cpufreq_cpu_put_sysfs(struct cpufreq_policy *data)
 253 {
 254         __cpufreq_cpu_put(data, true);
 255 }
 256
 257 /*********************************************************************
 258  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 259  *********************************************************************/
 260
 261 /**
 262  * adjust_jiffies - adjust the system "loops_per_jiffy"
 263  *
 264  * This function alters the system "loops_per_jiffy" for the clock
 265  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 266  * systems as each CPU might be scaled differently. So, use the arch
 267  * per-CPU loops_per_jiffy value wherever possible.
 268  */
 269 #ifndef CONFIG_SMP
 270 static unsigned long l_p_j_ref;
 271 static unsigned int  l_p_j_ref_freq;
 272
 273 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 274 {
 275         if (ci->flags & CPUFREQ_CONST_LOOPS)
 276                 return;
 277
 278         if (!l_p_j_ref_freq) {
 279                 l_p_j_ref = loops_per_jiffy;
 280                 l_p_j_ref_freq = ci->old;
 281                 pr_debug("saving %lu as reference value for loops_per_jiffy; "
 282                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 283         }
 284         if ((val == CPUFREQ_POSTCHANGE  && ci->old != ci->new) ||
 285             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 286                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 287                                                                 ci->new);
 288                 pr_debug("scaling loops_per_jiffy to %lu "
 289                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 290         }
 291 }
 292 #else
 293 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 294 {
 295         return;
 296 }
 297 #endif
 298
 299
 300 void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 301                 struct cpufreq_freqs *freqs, unsigned int state)
 302 {
 303         BUG_ON(irqs_disabled());
 304
 305         if (cpufreq_disabled())
 306                 return;
 307
 308         freqs->flags = cpufreq_driver->flags;
 309         pr_debug("notification %u of frequency transition to %u kHz\n",
 310                 state, freqs->new);
 311
 312         switch (state) {
 313
 314         case CPUFREQ_PRECHANGE:
 315                 /* detect if the driver reported a value as "old frequency"
 316                  * which is not equal to what the cpufreq core thinks is
 317                  * "old frequency".
 318                  */
 319                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 320                         if ((policy) && (policy->cpu == freqs->cpu) &&
 321                             (policy->cur) && (policy->cur != freqs->old)) {
 322                                 pr_debug("Warning: CPU frequency is"
 323                                         " %u, cpufreq assumed %u kHz.\n",
 324                                         freqs->old, policy->cur);
 325                                 freqs->old = policy->cur;
 326                         }
 327                 }
 328                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 329                                 CPUFREQ_PRECHANGE, freqs);
 330                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 331                 break;
 332
 333         case CPUFREQ_POSTCHANGE:
 334                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 335                 pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new,
 336                         (unsigned long)freqs->cpu);
 337                 trace_cpu_frequency(freqs->new, freqs->cpu);
 338                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 339                                 CPUFREQ_POSTCHANGE, freqs);
 340                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 341                         policy->cur = freqs->new;
 342                 break;
 343         }
 344 }
 345 /**
 346  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 347  * on frequency transition.
 348  *
 349  * This function calls the transition notifiers and the "adjust_jiffies"
 350  * function. It is called twice on all CPU frequency changes that have
 351  * external effects.
 352  */
 353 void cpufreq_notify_transition(struct cpufreq_policy *policy,
 354                 struct cpufreq_freqs *freqs, unsigned int state)
 355 {
 356         for_each_cpu(freqs->cpu, policy->cpus)
 357                 __cpufreq_notify_transition(policy, freqs, state);
 358 }
 359 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 360
 361
 362
 363 /*********************************************************************
 364  *                          SYSFS INTERFACE                          *
 365  *********************************************************************/
 366
 367 static struct cpufreq_governor *__find_governor(const char *str_governor)
 368 {
 369         struct cpufreq_governor *t;
 370
 371         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 372                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 373                         return t;
 374
 375         return NULL;
 376 }
 377
 378 /**
 379  * cpufreq_parse_governor - parse a governor string
 380  */
 381 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 382                                 struct cpufreq_governor **governor)
 383 {
 384         int err = -EINVAL;
 385
 386         if (!cpufreq_driver)
 387                 goto out;
 388
 389         if (cpufreq_driver->setpolicy) {
 390                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 391                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 392                         err = 0;
 393                 } else if (!strnicmp(str_governor, "powersave",
 394                                                 CPUFREQ_NAME_LEN)) {
 395                         *policy = CPUFREQ_POLICY_POWERSAVE;
 396                         err = 0;
 397                 }
 398         } else if (cpufreq_driver->target) {
 399                 struct cpufreq_governor *t;
 400
 401                 mutex_lock(&cpufreq_governor_mutex);
 402
 403                 t = __find_governor(str_governor);
 404
 405                 if (t == NULL) {
 406                         int ret;
 407
 408                         mutex_unlock(&cpufreq_governor_mutex);
 409                         ret = request_module("cpufreq_%s", str_governor);
 410                         mutex_lock(&cpufreq_governor_mutex);
 411
 412                         if (ret == 0)
 413                                 t = __find_governor(str_governor);
 414                 }
 415
 416                 if (t != NULL) {
 417                         *governor = t;
 418                         err = 0;
 419                 }
 420
 421                 mutex_unlock(&cpufreq_governor_mutex);
 422         }
 423 out:
 424         return err;
 425 }
 426
 427
 428 /**
 429  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 430  * print out cpufreq information
 431  *
 432  * Write out information from cpufreq_driver->policy[cpu]; object must be
 433  * "unsigned int".
 434  */
 435
 436 #define show_one(file_name, object)                     \
 437 static ssize_t show_##file_name                         \
 438 (struct cpufreq_policy *policy, char *buf)              \
 439 {                                                       \
 440         return sprintf(buf, "%u\n", policy->object);    \
 441 }
 442
 443 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 444 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 445 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 446 show_one(scaling_min_freq, min);
 447 show_one(scaling_max_freq, max);
 448 show_one(scaling_cur_freq, cur);
 449
 450 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 451                                 struct cpufreq_policy *policy);
 452
 453 /**
 454  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 455  */
 456 #define store_one(file_name, object)                    \
 457 static ssize_t store_##file_name                                        \
 458 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 459 {                                                                       \
 460         unsigned int ret;                                               \
 461         struct cpufreq_policy new_policy;                               \
 462                                                                         \
 463         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 464         if (ret)                                                        \
 465                 return -EINVAL;                                         \
 466                                                                         \
 467         ret = sscanf(buf, "%u", &new_policy.object);                    \
 468         if (ret != 1)                                                   \
 469                 return -EINVAL;                                         \
 470                                                                         \
 471         ret = __cpufreq_set_policy(policy, &new_policy);                \
 472         policy->user_policy.object = policy->object;                    \
 473                                                                         \
 474         return ret ? ret : count;                                       \
 475 }
 476
 477 store_one(scaling_min_freq, min);
 478 store_one(scaling_max_freq, max);
 479
 480 /**
 481  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 482  */
 483 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 484                                         char *buf)
 485 {
 486         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 487         if (!cur_freq)
 488                 return sprintf(buf, "<unknown>");
 489         return sprintf(buf, "%u\n", cur_freq);
 490 }
 491
 492
 493 /**
 494  * show_scaling_governor - show the current policy for the specified CPU
 495  */
 496 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 497 {
 498         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 499                 return sprintf(buf, "powersave\n");
 500         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 501                 return sprintf(buf, "performance\n");
 502         else if (policy->governor)
 503                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 504                                 policy->governor->name);
 505         return -EINVAL;
 506 }
 507
 508
 509 /**
 510  * store_scaling_governor - store policy for the specified CPU
 511  */
 512 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 513                                         const char *buf, size_t count)
 514 {
 515         unsigned int ret;
 516         char    str_governor[16];
 517         struct cpufreq_policy new_policy;
 518
 519         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 520         if (ret)
 521                 return ret;
 522
 523         ret = sscanf(buf, "%15s", str_governor);
 524         if (ret != 1)
 525                 return -EINVAL;
 526
 527         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 528                                                 &new_policy.governor))
 529                 return -EINVAL;
 530
 531         /* Do not use cpufreq_set_policy here or the user_policy.max
 532            will be wrongly overridden */
 533         ret = __cpufreq_set_policy(policy, &new_policy);
 534
 535         policy->user_policy.policy = policy->policy;
 536         policy->user_policy.governor = policy->governor;
 537
 538         if (ret)
 539                 return ret;
 540         else
 541                 return count;
 542 }
 543
 544 /**
 545  * show_scaling_driver - show the cpufreq driver currently loaded
 546  */
 547 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 548 {
 549         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 550 }
 551
 552 /**
 553  * show_scaling_available_governors - show the available CPUfreq governors
 554  */
 555 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 556                                                 char *buf)
 557 {
 558         ssize_t i = 0;
 559         struct cpufreq_governor *t;
 560
 561         if (!cpufreq_driver->target) {
 562                 i += sprintf(buf, "performance powersave");
 563                 goto out;
 564         }
 565
 566         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 567                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 568                     - (CPUFREQ_NAME_LEN + 2)))
 569                         goto out;
 570                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 571         }
 572 out:
 573         i += sprintf(&buf[i], "\n");
 574         return i;
 575 }
 576
 577 static ssize_t show_cpus(const struct cpumask *mask, char *buf)
 578 {
 579         ssize_t i = 0;
 580         unsigned int cpu;
 581
 582         for_each_cpu(cpu, mask) {
 583                 if (i)
 584                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 585                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 586                 if (i >= (PAGE_SIZE - 5))
 587                         break;
 588         }
 589         i += sprintf(&buf[i], "\n");
 590         return i;
 591 }
 592
 593 /**
 594  * show_related_cpus - show the CPUs affected by each transition even if
 595  * hw coordination is in use
 596  */
 597 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 598 {
 599         return show_cpus(policy->related_cpus, buf);
 600 }
 601
 602 /**
 603  * show_affected_cpus - show the CPUs affected by each transition
 604  */
 605 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 606 {
 607         return show_cpus(policy->cpus, buf);
 608 }
 609
 610 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 611                                         const char *buf, size_t count)
 612 {
 613         unsigned int freq = 0;
 614         unsigned int ret;
 615
 616         if (!policy->governor || !policy->governor->store_setspeed)
 617                 return -EINVAL;
 618
 619         ret = sscanf(buf, "%u", &freq);
 620         if (ret != 1)
 621                 return -EINVAL;
 622
 623         policy->governor->store_setspeed(policy, freq);
 624
 625         return count;
 626 }
 627
 628 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 629 {
 630         if (!policy->governor || !policy->governor->show_setspeed)
 631                 return sprintf(buf, "<unsupported>\n");
 632
 633         return policy->governor->show_setspeed(policy, buf);
 634 }
 635
 636 /**
 637  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 638  */
 639 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 640 {
 641         unsigned int limit;
 642         int ret;
 643         if (cpufreq_driver->bios_limit) {
 644                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 645                 if (!ret)
 646                         return sprintf(buf, "%u\n", limit);
 647         }
 648         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 649 }
 650
 651 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 652 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 653 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 654 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 655 cpufreq_freq_attr_ro(scaling_available_governors);
 656 cpufreq_freq_attr_ro(scaling_driver);
 657 cpufreq_freq_attr_ro(scaling_cur_freq);
 658 cpufreq_freq_attr_ro(bios_limit);
 659 cpufreq_freq_attr_ro(related_cpus);
 660 cpufreq_freq_attr_ro(affected_cpus);
 661 cpufreq_freq_attr_rw(scaling_min_freq);
 662 cpufreq_freq_attr_rw(scaling_max_freq);
 663 cpufreq_freq_attr_rw(scaling_governor);
 664 cpufreq_freq_attr_rw(scaling_setspeed);
 665
 666 static struct attribute *default_attrs[] = {
 667         &cpuinfo_min_freq.attr,
 668         &cpuinfo_max_freq.attr,
 669         &cpuinfo_transition_latency.attr,
 670         &scaling_min_freq.attr,
 671         &scaling_max_freq.attr,
 672         &affected_cpus.attr,
 673         &related_cpus.attr,
 674         &scaling_governor.attr,
 675         &scaling_driver.attr,
 676         &scaling_available_governors.attr,
 677         &scaling_setspeed.attr,
 678         NULL
 679 };
 680
 681 struct kobject *cpufreq_global_kobject;
 682 EXPORT_SYMBOL(cpufreq_global_kobject);
 683
 684 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 685 #define to_attr(a) container_of(a, struct freq_attr, attr)
 686
 687 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 688 {
 689         struct cpufreq_policy *policy = to_policy(kobj);
 690         struct freq_attr *fattr = to_attr(attr);
 691         ssize_t ret = -EINVAL;
 692         policy = cpufreq_cpu_get_sysfs(policy->cpu);
 693         if (!policy)
 694                 goto no_policy;
 695
 696         if (lock_policy_rwsem_read(policy->cpu) < 0)
 697                 goto fail;
 698
 699         if (fattr->show)
 700                 ret = fattr->show(policy, buf);
 701         else
 702                 ret = -EIO;
 703
 704         unlock_policy_rwsem_read(policy->cpu);
 705 fail:
 706         cpufreq_cpu_put_sysfs(policy);
 707 no_policy:
 708         return ret;
 709 }
 710
 711 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 712                      const char *buf, size_t count)
 713 {
 714         struct cpufreq_policy *policy = to_policy(kobj);
 715         struct freq_attr *fattr = to_attr(attr);
 716         ssize_t ret = -EINVAL;
 717         policy = cpufreq_cpu_get_sysfs(policy->cpu);
 718         if (!policy)
 719                 goto no_policy;
 720
 721         if (lock_policy_rwsem_write(policy->cpu) < 0)
 722                 goto fail;
 723
 724         if (fattr->store)
 725                 ret = fattr->store(policy, buf, count);
 726         else
 727                 ret = -EIO;
 728
 729         unlock_policy_rwsem_write(policy->cpu);
 730 fail:
 731         cpufreq_cpu_put_sysfs(policy);
 732 no_policy:
 733         return ret;
 734 }
 735
 736 static void cpufreq_sysfs_release(struct kobject *kobj)
 737 {
 738         struct cpufreq_policy *policy = to_policy(kobj);
 739         pr_debug("last reference is dropped\n");
 740         complete(&policy->kobj_unregister);
 741 }
 742
 743 static const struct sysfs_ops sysfs_ops = {
 744         .show   = show,
 745         .store  = store,
 746 };
 747
 748 static struct kobj_type ktype_cpufreq = {
 749         .sysfs_ops      = &sysfs_ops,
 750         .default_attrs  = default_attrs,
 751         .release        = cpufreq_sysfs_release,
 752 };
 753
 754 /* symlink affected CPUs */
 755 static int cpufreq_add_dev_symlink(unsigned int cpu,
 756                                    struct cpufreq_policy *policy)
 757 {
 758         unsigned int j;
 759         int ret = 0;
 760
 761         for_each_cpu(j, policy->cpus) {
 762                 struct cpufreq_policy *managed_policy;
 763                 struct device *cpu_dev;
 764
 765                 if (j == cpu)
 766                         continue;
 767
 768                 pr_debug("CPU %u already managed, adding link\n", j);
 769                 managed_policy = cpufreq_cpu_get(cpu);
 770                 cpu_dev = get_cpu_device(j);
 771                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 772                                         "cpufreq");
 773                 if (ret) {
 774                         cpufreq_cpu_put(managed_policy);
 775                         return ret;
 776                 }
 777         }
 778         return ret;
 779 }
 780
 781 static int cpufreq_add_dev_interface(unsigned int cpu,
 782                                      struct cpufreq_policy *policy,
 783                                      struct device *dev)
 784 {
 785         struct cpufreq_policy new_policy;
 786         struct freq_attr **drv_attr;
 787         unsigned long flags;
 788         int ret = 0;
 789         unsigned int j;
 790
 791         /* prepare interface data */
 792         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 793                                    &dev->kobj, "cpufreq");
 794         if (ret)
 795                 return ret;
 796
 797         /* set up files for this cpu device */
 798         drv_attr = cpufreq_driver->attr;
 799         while ((drv_attr) && (*drv_attr)) {
 800                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 801                 if (ret)
 802                         goto err_out_kobj_put;
 803                 drv_attr++;
 804         }
 805         if (cpufreq_driver->get) {
 806                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 807                 if (ret)
 808                         goto err_out_kobj_put;
 809         }
 810         if (cpufreq_driver->target) {
 811                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 812                 if (ret)
 813                         goto err_out_kobj_put;
 814         }
 815         if (cpufreq_driver->bios_limit) {
 816                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 817                 if (ret)
 818                         goto err_out_kobj_put;
 819         }
 820
 821         write_lock_irqsave(&cpufreq_driver_lock, flags);
 822         for_each_cpu(j, policy->cpus) {
 823                 per_cpu(cpufreq_cpu_data, j) = policy;
 824                 per_cpu(cpufreq_policy_cpu, j) = policy->cpu;
 825         }
 826         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 827
 828         ret = cpufreq_add_dev_symlink(cpu, policy);
 829         if (ret)
 830                 goto err_out_kobj_put;
 831
 832         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 833         /* assure that the starting sequence is run in __cpufreq_set_policy */
 834         policy->governor = NULL;
 835
 836         /* set default policy */
 837         ret = __cpufreq_set_policy(policy, &new_policy);
 838         policy->user_policy.policy = policy->policy;
 839         policy->user_policy.governor = policy->governor;
 840
 841         if (ret) {
 842                 pr_debug("setting policy failed\n");
 843                 if (cpufreq_driver->exit)
 844                         cpufreq_driver->exit(policy);
 845         }
 846         return ret;
 847
 848 err_out_kobj_put:
 849         kobject_put(&policy->kobj);
 850         wait_for_completion(&policy->kobj_unregister);
 851         return ret;
 852 }
 853
 854 #ifdef CONFIG_HOTPLUG_CPU
 855 static int cpufreq_add_policy_cpu(unsigned int cpu, unsigned int sibling,
 856                                   struct device *dev)
 857 {
 858         struct cpufreq_policy *policy;
 859         int ret = 0, has_target = !!cpufreq_driver->target;
 860         unsigned long flags;
 861
 862         policy = cpufreq_cpu_get(sibling);
 863         WARN_ON(!policy);
 864
 865         if (has_target)
 866                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 867
 868         lock_policy_rwsem_write(sibling);
 869
 870         write_lock_irqsave(&cpufreq_driver_lock, flags);
 871
 872         cpumask_set_cpu(cpu, policy->cpus);
 873         per_cpu(cpufreq_policy_cpu, cpu) = policy->cpu;
 874         per_cpu(cpufreq_cpu_data, cpu) = policy;
 875         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 876
 877         unlock_policy_rwsem_write(sibling);
 878
 879         if (has_target) {
 880                 __cpufreq_governor(policy, CPUFREQ_GOV_START);
 881                 __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
 882         }
 883
 884         ret = sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
 885         if (ret) {
 886                 cpufreq_cpu_put(policy);
 887                 return ret;
 888         }
 889
 890         return 0;
 891 }
 892 #endif
 893
 894 /**
 895  * cpufreq_add_dev - add a CPU device
 896  *
 897  * Adds the cpufreq interface for a CPU device.
 898  *
 899  * The Oracle says: try running cpufreq registration/unregistration concurrently
 900  * with with cpu hotplugging and all hell will break loose. Tried to clean this
 901  * mess up, but more thorough testing is needed. - Mathieu
 902  */
 903 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
 904 {
 905         unsigned int j, cpu = dev->id;
 906         int ret = -ENOMEM;
 907         struct cpufreq_policy *policy;
 908         unsigned long flags;
 909 #ifdef CONFIG_HOTPLUG_CPU
 910         struct cpufreq_governor *gov;
 911         int sibling;
 912 #endif
 913
 914         if (cpu_is_offline(cpu))
 915                 return 0;
 916
 917         pr_debug("adding CPU %u\n", cpu);
 918
 919 #ifdef CONFIG_SMP
 920         /* check whether a different CPU already registered this
 921          * CPU because it is in the same boat. */
 922         policy = cpufreq_cpu_get(cpu);
 923         if (unlikely(policy)) {
 924                 cpufreq_cpu_put(policy);
 925                 return 0;
 926         }
 927
 928 #ifdef CONFIG_HOTPLUG_CPU
 929         /* Check if this cpu was hot-unplugged earlier and has siblings */
 930         read_lock_irqsave(&cpufreq_driver_lock, flags);
 931         for_each_online_cpu(sibling) {
 932                 struct cpufreq_policy *cp = per_cpu(cpufreq_cpu_data, sibling);
 933                 if (cp && cpumask_test_cpu(cpu, cp->related_cpus)) {
 934                         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 935                         return cpufreq_add_policy_cpu(cpu, sibling, dev);
 936                 }
 937         }
 938         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 939 #endif
 940 #endif
 941
 942         if (!try_module_get(cpufreq_driver->owner)) {
 943                 ret = -EINVAL;
 944                 goto module_out;
 945         }
 946
 947         policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
 948         if (!policy)
 949                 goto nomem_out;
 950
 951         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
 952                 goto err_free_policy;
 953
 954         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
 955                 goto err_free_cpumask;
 956
 957         policy->cpu = cpu;
 958         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
 959         cpumask_copy(policy->cpus, cpumask_of(cpu));
 960
 961         /* Initially set CPU itself as the policy_cpu */
 962         per_cpu(cpufreq_policy_cpu, cpu) = cpu;
 963
 964         init_completion(&policy->kobj_unregister);
 965         INIT_WORK(&policy->update, handle_update);
 966
 967         /* call driver. From then on the cpufreq must be able
 968          * to accept all calls to ->verify and ->setpolicy for this CPU
 969          */
 970         ret = cpufreq_driver->init(policy);
 971         if (ret) {
 972                 pr_debug("initialization failed\n");
 973                 goto err_set_policy_cpu;
 974         }
 975
 976         /* related cpus should atleast have policy->cpus */
 977         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
 978
 979         /*
 980          * affected cpus must always be the one, which are online. We aren't
 981          * managing offline cpus here.
 982          */
 983         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
 984
 985         policy->user_policy.min = policy->min;
 986         policy->user_policy.max = policy->max;
 987
 988         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
 989                                      CPUFREQ_START, policy);
 990
 991 #ifdef CONFIG_HOTPLUG_CPU
 992         gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
 993         if (gov) {
 994                 policy->governor = gov;
 995                 pr_debug("Restoring governor %s for cpu %d\n",
 996                        policy->governor->name, cpu);
 997         }
 998 #endif
 999
1000         ret = cpufreq_add_dev_interface(cpu, policy, dev);
1001         if (ret)
1002                 goto err_out_unregister;
1003
1004         kobject_uevent(&policy->kobj, KOBJ_ADD);
1005         module_put(cpufreq_driver->owner);
1006         pr_debug("initialization complete\n");
1007
1008         return 0;
1009
1010 err_out_unregister:
1011         write_lock_irqsave(&cpufreq_driver_lock, flags);
1012         for_each_cpu(j, policy->cpus)
1013                 per_cpu(cpufreq_cpu_data, j) = NULL;
1014         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1015
1016         kobject_put(&policy->kobj);
1017         wait_for_completion(&policy->kobj_unregister);
1018
1019 err_set_policy_cpu:
1020         per_cpu(cpufreq_policy_cpu, cpu) = -1;
1021         free_cpumask_var(policy->related_cpus);
1022 err_free_cpumask:
1023         free_cpumask_var(policy->cpus);
1024 err_free_policy:
1025         kfree(policy);
1026 nomem_out:
1027         module_put(cpufreq_driver->owner);
1028 module_out:
1029         return ret;
1030 }
1031
1032 static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1033 {
1034         int j;
1035
1036         policy->last_cpu = policy->cpu;
1037         policy->cpu = cpu;
1038
1039         for_each_cpu(j, policy->cpus)
1040                 per_cpu(cpufreq_policy_cpu, j) = cpu;
1041
1042 #ifdef CONFIG_CPU_FREQ_TABLE
1043         cpufreq_frequency_table_update_policy_cpu(policy);
1044 #endif
1045         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1046                         CPUFREQ_UPDATE_POLICY_CPU, policy);
1047 }
1048
1049 /**
1050  * __cpufreq_remove_dev - remove a CPU device
1051  *
1052  * Removes the cpufreq interface for a CPU device.
1053  * Caller should already have policy_rwsem in write mode for this CPU.
1054  * This routine frees the rwsem before returning.
1055  */
1056 static int __cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1057 {
1058         unsigned int cpu = dev->id, ret, cpus;
1059         unsigned long flags;
1060         struct cpufreq_policy *data;
1061         struct kobject *kobj;
1062         struct completion *cmp;
1063         struct device *cpu_dev;
1064
1065         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1066
1067         write_lock_irqsave(&cpufreq_driver_lock, flags);
1068
1069         data = per_cpu(cpufreq_cpu_data, cpu);
1070         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1071
1072         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1073
1074         if (!data) {
1075                 pr_debug("%s: No cpu_data found\n", __func__);
1076                 return -EINVAL;
1077         }
1078
1079         if (cpufreq_driver->target)
1080                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1081
1082 #ifdef CONFIG_HOTPLUG_CPU
1083         if (!cpufreq_driver->setpolicy)
1084                 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1085                         data->governor->name, CPUFREQ_NAME_LEN);
1086 #endif
1087
1088         WARN_ON(lock_policy_rwsem_write(cpu));
1089         cpus = cpumask_weight(data->cpus);
1090
1091         if (cpus > 1)
1092                 cpumask_clear_cpu(cpu, data->cpus);
1093         unlock_policy_rwsem_write(cpu);
1094
1095         if (cpu != data->cpu) {
1096                 sysfs_remove_link(&dev->kobj, "cpufreq");
1097         } else if (cpus > 1) {
1098                 /* first sibling now owns the new sysfs dir */
1099                 cpu_dev = get_cpu_device(cpumask_first(data->cpus));
1100                 sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1101                 ret = kobject_move(&data->kobj, &cpu_dev->kobj);
1102                 if (ret) {
1103                         pr_err("%s: Failed to move kobj: %d", __func__, ret);
1104
1105                         WARN_ON(lock_policy_rwsem_write(cpu));
1106                         cpumask_set_cpu(cpu, data->cpus);
1107
1108                         write_lock_irqsave(&cpufreq_driver_lock, flags);
1109                         per_cpu(cpufreq_cpu_data, cpu) = data;
1110                         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1111
1112                         unlock_policy_rwsem_write(cpu);
1113
1114                         ret = sysfs_create_link(&cpu_dev->kobj, &data->kobj,
1115                                         "cpufreq");
1116                         return -EINVAL;
1117                 }
1118
1119                 WARN_ON(lock_policy_rwsem_write(cpu));
1120                 update_policy_cpu(data, cpu_dev->id);
1121                 unlock_policy_rwsem_write(cpu);
1122                 pr_debug("%s: policy Kobject moved to cpu: %d from: %d\n",
1123                                 __func__, cpu_dev->id, cpu);
1124         }
1125
1126         /* If cpu is last user of policy, free policy */
1127         if (cpus == 1) {
1128                 if (cpufreq_driver->target)
1129                         __cpufreq_governor(data, CPUFREQ_GOV_POLICY_EXIT);
1130
1131                 lock_policy_rwsem_read(cpu);
1132                 kobj = &data->kobj;
1133                 cmp = &data->kobj_unregister;
1134                 unlock_policy_rwsem_read(cpu);
1135                 kobject_put(kobj);
1136
1137                 /* we need to make sure that the underlying kobj is actually
1138                  * not referenced anymore by anybody before we proceed with
1139                  * unloading.
1140                  */
1141                 pr_debug("waiting for dropping of refcount\n");
1142                 wait_for_completion(cmp);
1143                 pr_debug("wait complete\n");
1144
1145                 if (cpufreq_driver->exit)
1146                         cpufreq_driver->exit(data);
1147
1148                 free_cpumask_var(data->related_cpus);
1149                 free_cpumask_var(data->cpus);
1150                 kfree(data);
1151         } else {
1152                 pr_debug("%s: removing link, cpu: %d\n", __func__, cpu);
1153                 cpufreq_cpu_put(data);
1154                 if (cpufreq_driver->target) {
1155                         __cpufreq_governor(data, CPUFREQ_GOV_START);
1156                         __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1157                 }
1158         }
1159
1160         per_cpu(cpufreq_policy_cpu, cpu) = -1;
1161         return 0;
1162 }
1163
1164
1165 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1166 {
1167         unsigned int cpu = dev->id;
1168         int retval;
1169
1170         if (cpu_is_offline(cpu))
1171                 return 0;
1172
1173         retval = __cpufreq_remove_dev(dev, sif);
1174         return retval;
1175 }
1176
1177
1178 static void handle_update(struct work_struct *work)
1179 {
1180         struct cpufreq_policy *policy =
1181                 container_of(work, struct cpufreq_policy, update);
1182         unsigned int cpu = policy->cpu;
1183         pr_debug("handle_update for cpu %u called\n", cpu);
1184         cpufreq_update_policy(cpu);
1185 }
1186
1187 /**
1188  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1189  *      @cpu: cpu number
1190  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1191  *      @new_freq: CPU frequency the CPU actually runs at
1192  *
1193  *      We adjust to current frequency first, and need to clean up later.
1194  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1195  */
1196 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1197                                 unsigned int new_freq)
1198 {
1199         struct cpufreq_policy *policy;
1200         struct cpufreq_freqs freqs;
1201         unsigned long flags;
1202
1203
1204         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing "
1205                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1206
1207         freqs.old = old_freq;
1208         freqs.new = new_freq;
1209
1210         read_lock_irqsave(&cpufreq_driver_lock, flags);
1211         policy = per_cpu(cpufreq_cpu_data, cpu);
1212         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1213
1214         cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE);
1215         cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE);
1216 }
1217
1218
1219 /**
1220  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1221  * @cpu: CPU number
1222  *
1223  * This is the last known freq, without actually getting it from the driver.
1224  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1225  */
1226 unsigned int cpufreq_quick_get(unsigned int cpu)
1227 {
1228         struct cpufreq_policy *policy;
1229         unsigned int ret_freq = 0;
1230
1231         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1232                 return cpufreq_driver->get(cpu);
1233
1234         policy = cpufreq_cpu_get(cpu);
1235         if (policy) {
1236                 ret_freq = policy->cur;
1237                 cpufreq_cpu_put(policy);
1238         }
1239
1240         return ret_freq;
1241 }
1242 EXPORT_SYMBOL(cpufreq_quick_get);
1243
1244 /**
1245  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1246  * @cpu: CPU number
1247  *
1248  * Just return the max possible frequency for a given CPU.
1249  */
1250 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1251 {
1252         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1253         unsigned int ret_freq = 0;
1254
1255         if (policy) {
1256                 ret_freq = policy->max;
1257                 cpufreq_cpu_put(policy);
1258         }
1259
1260         return ret_freq;
1261 }
1262 EXPORT_SYMBOL(cpufreq_quick_get_max);
1263
1264
1265 static unsigned int __cpufreq_get(unsigned int cpu)
1266 {
1267         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1268         unsigned int ret_freq = 0;
1269
1270         if (!cpufreq_driver->get)
1271                 return ret_freq;
1272
1273         ret_freq = cpufreq_driver->get(cpu);
1274
1275         if (ret_freq && policy->cur &&
1276                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1277                 /* verify no discrepancy between actual and
1278                                         saved value exists */
1279                 if (unlikely(ret_freq != policy->cur)) {
1280                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1281                         schedule_work(&policy->update);
1282                 }
1283         }
1284
1285         return ret_freq;
1286 }
1287
1288 /**
1289  * cpufreq_get - get the current CPU frequency (in kHz)
1290  * @cpu: CPU number
1291  *
1292  * Get the CPU current (static) CPU frequency
1293  */
1294 unsigned int cpufreq_get(unsigned int cpu)
1295 {
1296         unsigned int ret_freq = 0;
1297         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1298
1299         if (!policy)
1300                 goto out;
1301
1302         if (unlikely(lock_policy_rwsem_read(cpu)))
1303                 goto out_policy;
1304
1305         ret_freq = __cpufreq_get(cpu);
1306
1307         unlock_policy_rwsem_read(cpu);
1308
1309 out_policy:
1310         cpufreq_cpu_put(policy);
1311 out:
1312         return ret_freq;
1313 }
1314 EXPORT_SYMBOL(cpufreq_get);
1315
1316 static struct subsys_interface cpufreq_interface = {
1317         .name           = "cpufreq",
1318         .subsys         = &cpu_subsys,
1319         .add_dev        = cpufreq_add_dev,
1320         .remove_dev     = cpufreq_remove_dev,
1321 };
1322
1323
1324 /**
1325  * cpufreq_bp_suspend - Prepare the boot CPU for system suspend.
1326  *
1327  * This function is only executed for the boot processor.  The other CPUs
1328  * have been put offline by means of CPU hotplug.
1329  */
1330 static int cpufreq_bp_suspend(void)
1331 {
1332         int ret = 0;
1333
1334         int cpu = smp_processor_id();
1335         struct cpufreq_policy *cpu_policy;
1336
1337         pr_debug("suspending cpu %u\n", cpu);
1338
1339         /* If there's no policy for the boot CPU, we have nothing to do. */
1340         cpu_policy = cpufreq_cpu_get(cpu);
1341         if (!cpu_policy)
1342                 return 0;
1343
1344         if (cpufreq_driver->suspend) {
1345                 ret = cpufreq_driver->suspend(cpu_policy);
1346                 if (ret)
1347                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1348                                         "step on CPU %u\n", cpu_policy->cpu);
1349         }
1350
1351         cpufreq_cpu_put(cpu_policy);
1352         return ret;
1353 }
1354
1355 /**
1356  * cpufreq_bp_resume - Restore proper frequency handling of the boot CPU.
1357  *
1358  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1359  *      2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1360  *          restored. It will verify that the current freq is in sync with
1361  *          what we believe it to be. This is a bit later than when it
1362  *          should be, but nonethteless it's better than calling
1363  *          cpufreq_driver->get() here which might re-enable interrupts...
1364  *
1365  * This function is only executed for the boot CPU.  The other CPUs have not
1366  * been turned on yet.
1367  */
1368 static void cpufreq_bp_resume(void)
1369 {
1370         int ret = 0;
1371
1372         int cpu = smp_processor_id();
1373         struct cpufreq_policy *cpu_policy;
1374
1375         pr_debug("resuming cpu %u\n", cpu);
1376
1377         /* If there's no policy for the boot CPU, we have nothing to do. */
1378         cpu_policy = cpufreq_cpu_get(cpu);
1379         if (!cpu_policy)
1380                 return;
1381
1382         if (cpufreq_driver->resume) {
1383                 ret = cpufreq_driver->resume(cpu_policy);
1384                 if (ret) {
1385                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1386                                         "step on CPU %u\n", cpu_policy->cpu);
1387                         goto fail;
1388                 }
1389         }
1390
1391         schedule_work(&cpu_policy->update);
1392
1393 fail:
1394         cpufreq_cpu_put(cpu_policy);
1395 }
1396
1397 static struct syscore_ops cpufreq_syscore_ops = {
1398         .suspend        = cpufreq_bp_suspend,
1399         .resume         = cpufreq_bp_resume,
1400 };
1401
1402 /**
1403  *      cpufreq_get_current_driver - return current driver's name
1404  *
1405  *      Return the name string of the currently loaded cpufreq driver
1406  *      or NULL, if none.
1407  */
1408 const char *cpufreq_get_current_driver(void)
1409 {
1410         if (cpufreq_driver)
1411                 return cpufreq_driver->name;
1412
1413         return NULL;
1414 }
1415 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1416
1417 /*********************************************************************
1418  *                     NOTIFIER LISTS INTERFACE                      *
1419  *********************************************************************/
1420
1421 /**
1422  *      cpufreq_register_notifier - register a driver with cpufreq
1423  *      @nb: notifier function to register
1424  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1425  *
1426  *      Add a driver to one of two lists: either a list of drivers that
1427  *      are notified about clock rate changes (once before and once after
1428  *      the transition), or a list of drivers that are notified about
1429  *      changes in cpufreq policy.
1430  *
1431  *      This function may sleep, and has the same return conditions as
1432  *      blocking_notifier_chain_register.
1433  */
1434 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1435 {
1436         int ret;
1437
1438         if (cpufreq_disabled())
1439                 return -EINVAL;
1440
1441         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1442
1443         switch (list) {
1444         case CPUFREQ_TRANSITION_NOTIFIER:
1445                 ret = srcu_notifier_chain_register(
1446                                 &cpufreq_transition_notifier_list, nb);
1447                 break;
1448         case CPUFREQ_POLICY_NOTIFIER:
1449                 ret = blocking_notifier_chain_register(
1450                                 &cpufreq_policy_notifier_list, nb);
1451                 break;
1452         default:
1453                 ret = -EINVAL;
1454         }
1455
1456         return ret;
1457 }
1458 EXPORT_SYMBOL(cpufreq_register_notifier);
1459
1460
1461 /**
1462  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1463  *      @nb: notifier block to be unregistered
1464  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1465  *
1466  *      Remove a driver from the CPU frequency notifier list.
1467  *
1468  *      This function may sleep, and has the same return conditions as
1469  *      blocking_notifier_chain_unregister.
1470  */
1471 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1472 {
1473         int ret;
1474
1475         if (cpufreq_disabled())
1476                 return -EINVAL;
1477
1478         switch (list) {
1479         case CPUFREQ_TRANSITION_NOTIFIER:
1480                 ret = srcu_notifier_chain_unregister(
1481                                 &cpufreq_transition_notifier_list, nb);
1482                 break;
1483         case CPUFREQ_POLICY_NOTIFIER:
1484                 ret = blocking_notifier_chain_unregister(
1485                                 &cpufreq_policy_notifier_list, nb);
1486                 break;
1487         default:
1488                 ret = -EINVAL;
1489         }
1490
1491         return ret;
1492 }
1493 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1494
1495
1496 /*********************************************************************
1497  *                              GOVERNORS                            *
1498  *********************************************************************/
1499
1500
1501 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1502                             unsigned int target_freq,
1503                             unsigned int relation)
1504 {
1505         int retval = -EINVAL;
1506         unsigned int old_target_freq = target_freq;
1507
1508         if (cpufreq_disabled())
1509                 return -ENODEV;
1510
1511         /* Make sure that target_freq is within supported range */
1512         if (target_freq > policy->max)
1513                 target_freq = policy->max;
1514         if (target_freq < policy->min)
1515                 target_freq = policy->min;
1516
1517         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1518                         policy->cpu, target_freq, relation, old_target_freq);
1519
1520         if (target_freq == policy->cur)
1521                 return 0;
1522
1523         if (cpufreq_driver->target)
1524                 retval = cpufreq_driver->target(policy, target_freq, relation);
1525
1526         return retval;
1527 }
1528 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1529
1530 int cpufreq_driver_target(struct cpufreq_policy *policy,
1531                           unsigned int target_freq,
1532                           unsigned int relation)
1533 {
1534         int ret = -EINVAL;
1535
1536         policy = cpufreq_cpu_get(policy->cpu);
1537         if (!policy)
1538                 goto no_policy;
1539
1540         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1541                 goto fail;
1542
1543         ret = __cpufreq_driver_target(policy, target_freq, relation);
1544
1545         unlock_policy_rwsem_write(policy->cpu);
1546
1547 fail:
1548         cpufreq_cpu_put(policy);
1549 no_policy:
1550         return ret;
1551 }
1552 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1553
1554 int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1555 {
1556         int ret = 0;
1557
1558         if (cpufreq_disabled())
1559                 return ret;
1560
1561         if (!cpufreq_driver->getavg)
1562                 return 0;
1563
1564         policy = cpufreq_cpu_get(policy->cpu);
1565         if (!policy)
1566                 return -EINVAL;
1567
1568         ret = cpufreq_driver->getavg(policy, cpu);
1569
1570         cpufreq_cpu_put(policy);
1571         return ret;
1572 }
1573 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1574
1575 /*
1576  * when "event" is CPUFREQ_GOV_LIMITS
1577  */
1578
1579 static int __cpufreq_governor(struct cpufreq_policy *policy,
1580                                         unsigned int event)
1581 {
1582         int ret;
1583
1584         /* Only must be defined when default governor is known to have latency
1585            restrictions, like e.g. conservative or ondemand.
1586            That this is the case is already ensured in Kconfig
1587         */
1588 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1589         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1590 #else
1591         struct cpufreq_governor *gov = NULL;
1592 #endif
1593
1594         if (policy->governor->max_transition_latency &&
1595             policy->cpuinfo.transition_latency >
1596             policy->governor->max_transition_latency) {
1597                 if (!gov)
1598                         return -EINVAL;
1599                 else {
1600                         printk(KERN_WARNING "%s governor failed, too long"
1601                                " transition latency of HW, fallback"
1602                                " to %s governor\n",
1603                                policy->governor->name,
1604                                gov->name);
1605                         policy->governor = gov;
1606                 }
1607         }
1608
1609         if (!try_module_get(policy->governor->owner))
1610                 return -EINVAL;
1611
1612         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1613                                                 policy->cpu, event);
1614         ret = policy->governor->governor(policy, event);
1615
1616         if (!ret) {
1617                 if (event == CPUFREQ_GOV_POLICY_INIT)
1618                         policy->governor->initialized++;
1619                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1620                         policy->governor->initialized--;
1621         }
1622
1623         /* we keep one module reference alive for
1624                         each CPU governed by this CPU */
1625         if ((event != CPUFREQ_GOV_START) || ret)
1626                 module_put(policy->governor->owner);
1627         if ((event == CPUFREQ_GOV_STOP) && !ret)
1628                 module_put(policy->governor->owner);
1629
1630         return ret;
1631 }
1632
1633
1634 int cpufreq_register_governor(struct cpufreq_governor *governor)
1635 {
1636         int err;
1637
1638         if (!governor)
1639                 return -EINVAL;
1640
1641         if (cpufreq_disabled())
1642                 return -ENODEV;
1643
1644         mutex_lock(&cpufreq_governor_mutex);
1645
1646         governor->initialized = 0;
1647         err = -EBUSY;
1648         if (__find_governor(governor->name) == NULL) {
1649                 err = 0;
1650                 list_add(&governor->governor_list, &cpufreq_governor_list);
1651         }
1652
1653         mutex_unlock(&cpufreq_governor_mutex);
1654         return err;
1655 }
1656 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1657
1658
1659 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1660 {
1661 #ifdef CONFIG_HOTPLUG_CPU
1662         int cpu;
1663 #endif
1664
1665         if (!governor)
1666                 return;
1667
1668         if (cpufreq_disabled())
1669                 return;
1670
1671 #ifdef CONFIG_HOTPLUG_CPU
1672         for_each_present_cpu(cpu) {
1673                 if (cpu_online(cpu))
1674                         continue;
1675                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
1676                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
1677         }
1678 #endif
1679
1680         mutex_lock(&cpufreq_governor_mutex);
1681         list_del(&governor->governor_list);
1682         mutex_unlock(&cpufreq_governor_mutex);
1683         return;
1684 }
1685 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1686
1687
1688
1689 /*********************************************************************
1690  *                          POLICY INTERFACE                         *
1691  *********************************************************************/
1692
1693 /**
1694  * cpufreq_get_policy - get the current cpufreq_policy
1695  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1696  *      is written
1697  *
1698  * Reads the current cpufreq policy.
1699  */
1700 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1701 {
1702         struct cpufreq_policy *cpu_policy;
1703         if (!policy)
1704                 return -EINVAL;
1705
1706         cpu_policy = cpufreq_cpu_get(cpu);
1707         if (!cpu_policy)
1708                 return -EINVAL;
1709
1710         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1711
1712         cpufreq_cpu_put(cpu_policy);
1713         return 0;
1714 }
1715 EXPORT_SYMBOL(cpufreq_get_policy);
1716
1717
1718 /*
1719  * data   : current policy.
1720  * policy : policy to be set.
1721  */
1722 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1723                                 struct cpufreq_policy *policy)
1724 {
1725         int ret = 0, failed = 1;
1726
1727         pr_debug("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1728                 policy->min, policy->max);
1729
1730         memcpy(&policy->cpuinfo, &data->cpuinfo,
1731                                 sizeof(struct cpufreq_cpuinfo));
1732
1733         if (policy->min > data->max || policy->max < data->min) {
1734                 ret = -EINVAL;
1735                 goto error_out;
1736         }
1737
1738         /* verify the cpu speed can be set within this limit */
1739         ret = cpufreq_driver->verify(policy);
1740         if (ret)
1741                 goto error_out;
1742
1743         /* adjust if necessary - all reasons */
1744         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1745                         CPUFREQ_ADJUST, policy);
1746
1747         /* adjust if necessary - hardware incompatibility*/
1748         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1749                         CPUFREQ_INCOMPATIBLE, policy);
1750
1751         /* verify the cpu speed can be set within this limit,
1752            which might be different to the first one */
1753         ret = cpufreq_driver->verify(policy);
1754         if (ret)
1755                 goto error_out;
1756
1757         /* notification of the new policy */
1758         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1759                         CPUFREQ_NOTIFY, policy);
1760
1761         data->min = policy->min;
1762         data->max = policy->max;
1763
1764         pr_debug("new min and max freqs are %u - %u kHz\n",
1765                                         data->min, data->max);
1766
1767         if (cpufreq_driver->setpolicy) {
1768                 data->policy = policy->policy;
1769                 pr_debug("setting range\n");
1770                 ret = cpufreq_driver->setpolicy(policy);
1771         } else {
1772                 if (policy->governor != data->governor) {
1773                         /* save old, working values */
1774                         struct cpufreq_governor *old_gov = data->governor;
1775
1776                         pr_debug("governor switch\n");
1777
1778                         /* end old governor */
1779                         if (data->governor) {
1780                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1781                                 unlock_policy_rwsem_write(policy->cpu);
1782                                 __cpufreq_governor(data,
1783                                                 CPUFREQ_GOV_POLICY_EXIT);
1784                                 lock_policy_rwsem_write(policy->cpu);
1785                         }
1786
1787                         /* start new governor */
1788                         data->governor = policy->governor;
1789                         if (!__cpufreq_governor(data, CPUFREQ_GOV_POLICY_INIT)) {
1790                                 if (!__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1791                                         failed = 0;
1792                                 } else {
1793                                         unlock_policy_rwsem_write(policy->cpu);
1794                                         __cpufreq_governor(data,
1795                                                         CPUFREQ_GOV_POLICY_EXIT);
1796                                         lock_policy_rwsem_write(policy->cpu);
1797                                 }
1798                         }
1799
1800                         if (failed) {
1801                                 /* new governor failed, so re-start old one */
1802                                 pr_debug("starting governor %s failed\n",
1803                                                         data->governor->name);
1804                                 if (old_gov) {
1805                                         data->governor = old_gov;
1806                                         __cpufreq_governor(data,
1807                                                         CPUFREQ_GOV_POLICY_INIT);
1808                                         __cpufreq_governor(data,
1809                                                            CPUFREQ_GOV_START);
1810                                 }
1811                                 ret = -EINVAL;
1812                                 goto error_out;
1813                         }
1814                         /* might be a policy change, too, so fall through */
1815                 }
1816                 pr_debug("governor: change or update limits\n");
1817                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1818         }
1819
1820 error_out:
1821         return ret;
1822 }
1823
1824 /**
1825  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1826  *      @cpu: CPU which shall be re-evaluated
1827  *
1828  *      Useful for policy notifiers which have different necessities
1829  *      at different times.
1830  */
1831 int cpufreq_update_policy(unsigned int cpu)
1832 {
1833         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1834         struct cpufreq_policy policy;
1835         int ret;
1836
1837         if (!data) {
1838                 ret = -ENODEV;
1839                 goto no_policy;
1840         }
1841
1842         if (unlikely(lock_policy_rwsem_write(cpu))) {
1843                 ret = -EINVAL;
1844                 goto fail;
1845         }
1846
1847         pr_debug("updating policy for CPU %u\n", cpu);
1848         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1849         policy.min = data->user_policy.min;
1850         policy.max = data->user_policy.max;
1851         policy.policy = data->user_policy.policy;
1852         policy.governor = data->user_policy.governor;
1853
1854         /* BIOS might change freq behind our back
1855           -> ask driver for current freq and notify governors about a change */
1856         if (cpufreq_driver->get) {
1857                 policy.cur = cpufreq_driver->get(cpu);
1858                 if (!data->cur) {
1859                         pr_debug("Driver did not initialize current freq");
1860                         data->cur = policy.cur;
1861                 } else {
1862                         if (data->cur != policy.cur && cpufreq_driver->target)
1863                                 cpufreq_out_of_sync(cpu, data->cur,
1864                                                                 policy.cur);
1865                 }
1866         }
1867
1868         ret = __cpufreq_set_policy(data, &policy);
1869
1870         unlock_policy_rwsem_write(cpu);
1871
1872 fail:
1873         cpufreq_cpu_put(data);
1874 no_policy:
1875         return ret;
1876 }
1877 EXPORT_SYMBOL(cpufreq_update_policy);
1878
1879 static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1880                                         unsigned long action, void *hcpu)
1881 {
1882         unsigned int cpu = (unsigned long)hcpu;
1883         struct device *dev;
1884
1885         dev = get_cpu_device(cpu);
1886         if (dev) {
1887                 switch (action) {
1888                 case CPU_ONLINE:
1889                 case CPU_ONLINE_FROZEN:
1890                         cpufreq_add_dev(dev, NULL);
1891                         break;
1892                 case CPU_DOWN_PREPARE:
1893                 case CPU_DOWN_PREPARE_FROZEN:
1894                         __cpufreq_remove_dev(dev, NULL);
1895                         break;
1896                 case CPU_DOWN_FAILED:
1897                 case CPU_DOWN_FAILED_FROZEN:
1898                         cpufreq_add_dev(dev, NULL);
1899                         break;
1900                 }
1901         }
1902         return NOTIFY_OK;
1903 }
1904
1905 static struct notifier_block __refdata cpufreq_cpu_notifier = {
1906     .notifier_call = cpufreq_cpu_callback,
1907 };
1908
1909 /*********************************************************************
1910  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1911  *********************************************************************/
1912
1913 /**
1914  * cpufreq_register_driver - register a CPU Frequency driver
1915  * @driver_data: A struct cpufreq_driver containing the values#
1916  * submitted by the CPU Frequency driver.
1917  *
1918  *   Registers a CPU Frequency driver to this core code. This code
1919  * returns zero on success, -EBUSY when another driver got here first
1920  * (and isn't unregistered in the meantime).
1921  *
1922  */
1923 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1924 {
1925         unsigned long flags;
1926         int ret;
1927
1928         if (cpufreq_disabled())
1929                 return -ENODEV;
1930
1931         if (!driver_data || !driver_data->verify || !driver_data->init ||
1932             ((!driver_data->setpolicy) && (!driver_data->target)))
1933                 return -EINVAL;
1934
1935         pr_debug("trying to register driver %s\n", driver_data->name);
1936
1937         if (driver_data->setpolicy)
1938                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1939
1940         write_lock_irqsave(&cpufreq_driver_lock, flags);
1941         if (cpufreq_driver) {
1942                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1943                 return -EBUSY;
1944         }
1945         cpufreq_driver = driver_data;
1946         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1947
1948         ret = subsys_interface_register(&cpufreq_interface);
1949         if (ret)
1950                 goto err_null_driver;
1951
1952         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
1953                 int i;
1954                 ret = -ENODEV;
1955
1956                 /* check for at least one working CPU */
1957                 for (i = 0; i < nr_cpu_ids; i++)
1958                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
1959                                 ret = 0;
1960                                 break;
1961                         }
1962
1963                 /* if all ->init() calls failed, unregister */
1964                 if (ret) {
1965                         pr_debug("no CPU initialized for driver %s\n",
1966                                                         driver_data->name);
1967                         goto err_if_unreg;
1968                 }
1969         }
1970
1971         register_hotcpu_notifier(&cpufreq_cpu_notifier);
1972         pr_debug("driver %s up and running\n", driver_data->name);
1973
1974         return 0;
1975 err_if_unreg:
1976         subsys_interface_unregister(&cpufreq_interface);
1977 err_null_driver:
1978         write_lock_irqsave(&cpufreq_driver_lock, flags);
1979         cpufreq_driver = NULL;
1980         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1981         return ret;
1982 }
1983 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
1984
1985
1986 /**
1987  * cpufreq_unregister_driver - unregister the current CPUFreq driver
1988  *
1989  *    Unregister the current CPUFreq driver. Only call this if you have
1990  * the right to do so, i.e. if you have succeeded in initialising before!
1991  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
1992  * currently not initialised.
1993  */
1994 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
1995 {
1996         unsigned long flags;
1997
1998         if (!cpufreq_driver || (driver != cpufreq_driver))
1999                 return -EINVAL;
2000
2001         pr_debug("unregistering driver %s\n", driver->name);
2002
2003         subsys_interface_unregister(&cpufreq_interface);
2004         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2005
2006         write_lock_irqsave(&cpufreq_driver_lock, flags);
2007         cpufreq_driver = NULL;
2008         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2009
2010         return 0;
2011 }
2012 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2013
2014 static int __init cpufreq_core_init(void)
2015 {
2016         int cpu;
2017
2018         if (cpufreq_disabled())
2019                 return -ENODEV;
2020
2021         for_each_possible_cpu(cpu) {
2022                 per_cpu(cpufreq_policy_cpu, cpu) = -1;
2023                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
2024         }
2025
2026         cpufreq_global_kobject = kobject_create_and_add("cpufreq", &cpu_subsys.dev_root->kobj);
2027         BUG_ON(!cpufreq_global_kobject);
2028         register_syscore_ops(&cpufreq_syscore_ops);
2029
2030         return 0;
2031 }
2032 core_initcall(cpufreq_core_init);