drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *
   7  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   8  *      Added handling for CPU hotplug
   9  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  10  *      Fix handling for CPU hotplug -- affected CPUs
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License version 2 as
  14  * published by the Free Software Foundation.
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/kernel.h>
  21 #include <linux/module.h>
  22 #include <linux/init.h>
  23 #include <linux/notifier.h>
  24 #include <linux/cpufreq.h>
  25 #include <linux/delay.h>
  26 #include <linux/interrupt.h>
  27 #include <linux/spinlock.h>
  28 #include <linux/device.h>
  29 #include <linux/slab.h>
  30 #include <linux/cpu.h>
  31 #include <linux/completion.h>
  32 #include <linux/mutex.h>
  33 #include <linux/syscore_ops.h>
  34 #include <linux/suspend.h>
  35 #include <linux/tick.h>
  36
  37 #include <trace/events/power.h>
  38
  39 /**
  40  * The "cpufreq driver" - the arch- or hardware-dependent low
  41  * level driver of CPUFreq support, and its spinlock. This lock
  42  * also protects the cpufreq_cpu_data array.
  43  */
  44 static struct cpufreq_driver *cpufreq_driver;
  45 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  46 #ifdef CONFIG_HOTPLUG_CPU
  47 /* This one keeps track of the previously set governor of a removed CPU */
  48 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  49 #endif
  50 static DEFINE_RWLOCK(cpufreq_driver_lock);
  51 static DEFINE_MUTEX(cpufreq_governor_lock);
  52
  53 /* Flag to suspend/resume CPUFreq governors */
  54 static bool cpufreq_suspended;
  55
  56 static inline bool has_target(void)
  57 {
  58         return cpufreq_driver->target;
  59 }
  60
  61 /*
  62  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  63  * all cpufreq/hotplug/workqueue/etc related lock issues.
  64  *
  65  * The rules for this semaphore:
  66  * - Any routine that wants to read from the policy structure will
  67  *   do a down_read on this semaphore.
  68  * - Any routine that will write to the policy structure and/or may take away
  69  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  70  *   mode before doing so.
  71  *
  72  * Additional rules:
  73  * - Governor routines that can be called in cpufreq hotplug path should not
  74  *   take this sem as top level hotplug notifier handler takes this.
  75  * - Lock should not be held across
  76  *     __cpufreq_governor(data, CPUFREQ_GOV_STOP);
  77  */
  78 static DEFINE_PER_CPU(int, cpufreq_policy_cpu);
  79 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  80
  81 #define lock_policy_rwsem(mode, cpu)                                    \
  82 static int lock_policy_rwsem_##mode(int cpu)                            \
  83 {                                                                       \
  84         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);              \
  85         BUG_ON(policy_cpu == -1);                                       \
  86         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  87                                                                         \
  88         return 0;                                                       \
  89 }
  90
  91 lock_policy_rwsem(read, cpu);
  92 lock_policy_rwsem(write, cpu);
  93
  94 #define unlock_policy_rwsem(mode, cpu)                                  \
  95 static void unlock_policy_rwsem_##mode(int cpu)                         \
  96 {                                                                       \
  97         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);              \
  98         BUG_ON(policy_cpu == -1);                                       \
  99         up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));              \
 100 }
 101
 102 unlock_policy_rwsem(read, cpu);
 103 unlock_policy_rwsem(write, cpu);
 104
 105 /* internal prototypes */
 106 static int __cpufreq_governor(struct cpufreq_policy *policy,
 107                 unsigned int event);
 108 static unsigned int __cpufreq_get(unsigned int cpu);
 109 static void handle_update(struct work_struct *work);
 110
 111 /**
 112  * Two notifier lists: the "policy" list is involved in the
 113  * validation process for a new CPU frequency policy; the
 114  * "transition" list for kernel code that needs to handle
 115  * changes to devices when the CPU clock speed changes.
 116  * The mutex locks both lists.
 117  */
 118 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 119 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 120
 121 static bool init_cpufreq_transition_notifier_list_called;
 122 static int __init init_cpufreq_transition_notifier_list(void)
 123 {
 124         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 125         init_cpufreq_transition_notifier_list_called = true;
 126         return 0;
 127 }
 128 pure_initcall(init_cpufreq_transition_notifier_list);
 129
 130 static int off __read_mostly;
 131 static int cpufreq_disabled(void)
 132 {
 133         return off;
 134 }
 135 void disable_cpufreq(void)
 136 {
 137         off = 1;
 138 }
 139 static LIST_HEAD(cpufreq_governor_list);
 140 static DEFINE_MUTEX(cpufreq_governor_mutex);
 141
 142 bool have_governor_per_policy(void)
 143 {
 144         return cpufreq_driver->have_governor_per_policy;
 145 }
 146
 147 static struct cpufreq_policy *__cpufreq_cpu_get(unsigned int cpu, bool sysfs)
 148 {
 149         struct cpufreq_policy *data;
 150         unsigned long flags;
 151
 152         if (cpu >= nr_cpu_ids)
 153                 goto err_out;
 154
 155         /* get the cpufreq driver */
 156         read_lock_irqsave(&cpufreq_driver_lock, flags);
 157
 158         if (!cpufreq_driver)
 159                 goto err_out_unlock;
 160
 161         if (!try_module_get(cpufreq_driver->owner))
 162                 goto err_out_unlock;
 163
 164
 165         /* get the CPU */
 166         data = per_cpu(cpufreq_cpu_data, cpu);
 167
 168         if (!data)
 169                 goto err_out_put_module;
 170
 171         if (!sysfs && !kobject_get(&data->kobj))
 172                 goto err_out_put_module;
 173
 174         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 175         return data;
 176
 177 err_out_put_module:
 178         module_put(cpufreq_driver->owner);
 179 err_out_unlock:
 180         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 181 err_out:
 182         return NULL;
 183 }
 184
 185 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 186 {
 187         if (cpufreq_disabled())
 188                 return NULL;
 189
 190         return __cpufreq_cpu_get(cpu, false);
 191 }
 192 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 193
 194 static struct cpufreq_policy *cpufreq_cpu_get_sysfs(unsigned int cpu)
 195 {
 196         return __cpufreq_cpu_get(cpu, true);
 197 }
 198
 199 static void __cpufreq_cpu_put(struct cpufreq_policy *data, bool sysfs)
 200 {
 201         if (!sysfs)
 202                 kobject_put(&data->kobj);
 203         module_put(cpufreq_driver->owner);
 204 }
 205
 206 void cpufreq_cpu_put(struct cpufreq_policy *data)
 207 {
 208         if (cpufreq_disabled())
 209                 return;
 210
 211         __cpufreq_cpu_put(data, false);
 212 }
 213 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 214
 215 static void cpufreq_cpu_put_sysfs(struct cpufreq_policy *data)
 216 {
 217         __cpufreq_cpu_put(data, true);
 218 }
 219
 220 /*********************************************************************
 221  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 222  *********************************************************************/
 223
 224 /**
 225  * adjust_jiffies - adjust the system "loops_per_jiffy"
 226  *
 227  * This function alters the system "loops_per_jiffy" for the clock
 228  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 229  * systems as each CPU might be scaled differently. So, use the arch
 230  * per-CPU loops_per_jiffy value wherever possible.
 231  */
 232 #ifndef CONFIG_SMP
 233 static unsigned long l_p_j_ref;
 234 static unsigned int  l_p_j_ref_freq;
 235
 236 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 237 {
 238         if (ci->flags & CPUFREQ_CONST_LOOPS)
 239                 return;
 240
 241         if (!l_p_j_ref_freq) {
 242                 l_p_j_ref = loops_per_jiffy;
 243                 l_p_j_ref_freq = ci->old;
 244                 pr_debug("saving %lu as reference value for loops_per_jiffy; "
 245                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 246         }
 247         if ((val == CPUFREQ_POSTCHANGE  && ci->old != ci->new) ||
 248             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 249                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 250                                                                 ci->new);
 251                 pr_debug("scaling loops_per_jiffy to %lu "
 252                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 253         }
 254 }
 255 #else
 256 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 257 {
 258         return;
 259 }
 260 #endif
 261
 262
 263 void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 264                 struct cpufreq_freqs *freqs, unsigned int state)
 265 {
 266         BUG_ON(irqs_disabled());
 267
 268         if (cpufreq_disabled())
 269                 return;
 270
 271         freqs->flags = cpufreq_driver->flags;
 272         pr_debug("notification %u of frequency transition to %u kHz\n",
 273                 state, freqs->new);
 274
 275         switch (state) {
 276
 277         case CPUFREQ_PRECHANGE:
 278                 /* detect if the driver reported a value as "old frequency"
 279                  * which is not equal to what the cpufreq core thinks is
 280                  * "old frequency".
 281                  */
 282                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 283                         if ((policy) && (policy->cpu == freqs->cpu) &&
 284                             (policy->cur) && (policy->cur != freqs->old)) {
 285                                 pr_debug("Warning: CPU frequency is"
 286                                         " %u, cpufreq assumed %u kHz.\n",
 287                                         freqs->old, policy->cur);
 288                                 freqs->old = policy->cur;
 289                         }
 290                 }
 291                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 292                                 CPUFREQ_PRECHANGE, freqs);
 293                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 294                 break;
 295
 296         case CPUFREQ_POSTCHANGE:
 297                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 298                 pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new,
 299                         (unsigned long)freqs->cpu);
 300                 trace_cpu_frequency(freqs->new, freqs->cpu);
 301                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 302                                 CPUFREQ_POSTCHANGE, freqs);
 303                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 304                         policy->cur = freqs->new;
 305                 break;
 306         }
 307 }
 308 /**
 309  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 310  * on frequency transition.
 311  *
 312  * This function calls the transition notifiers and the "adjust_jiffies"
 313  * function. It is called twice on all CPU frequency changes that have
 314  * external effects.
 315  */
 316 void cpufreq_notify_transition(struct cpufreq_policy *policy,
 317                 struct cpufreq_freqs *freqs, unsigned int state)
 318 {
 319         for_each_cpu(freqs->cpu, policy->cpus)
 320                 __cpufreq_notify_transition(policy, freqs, state);
 321 }
 322 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 323
 324
 325
 326 /*********************************************************************
 327  *                          SYSFS INTERFACE                          *
 328  *********************************************************************/
 329
 330 static struct cpufreq_governor *__find_governor(const char *str_governor)
 331 {
 332         struct cpufreq_governor *t;
 333
 334         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 335                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 336                         return t;
 337
 338         return NULL;
 339 }
 340
 341 /**
 342  * cpufreq_parse_governor - parse a governor string
 343  */
 344 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 345                                 struct cpufreq_governor **governor)
 346 {
 347         int err = -EINVAL;
 348
 349         if (!cpufreq_driver)
 350                 goto out;
 351
 352         if (cpufreq_driver->setpolicy) {
 353                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 354                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 355                         err = 0;
 356                 } else if (!strnicmp(str_governor, "powersave",
 357                                                 CPUFREQ_NAME_LEN)) {
 358                         *policy = CPUFREQ_POLICY_POWERSAVE;
 359                         err = 0;
 360                 }
 361         } else if (cpufreq_driver->target) {
 362                 struct cpufreq_governor *t;
 363
 364                 mutex_lock(&cpufreq_governor_mutex);
 365
 366                 t = __find_governor(str_governor);
 367
 368                 if (t == NULL) {
 369                         int ret;
 370
 371                         mutex_unlock(&cpufreq_governor_mutex);
 372                         ret = request_module("cpufreq_%s", str_governor);
 373                         mutex_lock(&cpufreq_governor_mutex);
 374
 375                         if (ret == 0)
 376                                 t = __find_governor(str_governor);
 377                 }
 378
 379                 if (t != NULL) {
 380                         *governor = t;
 381                         err = 0;
 382                 }
 383
 384                 mutex_unlock(&cpufreq_governor_mutex);
 385         }
 386 out:
 387         return err;
 388 }
 389
 390
 391 /**
 392  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 393  * print out cpufreq information
 394  *
 395  * Write out information from cpufreq_driver->policy[cpu]; object must be
 396  * "unsigned int".
 397  */
 398
 399 #define show_one(file_name, object)                     \
 400 static ssize_t show_##file_name                         \
 401 (struct cpufreq_policy *policy, char *buf)              \
 402 {                                                       \
 403         return sprintf(buf, "%u\n", policy->object);    \
 404 }
 405
 406 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 407 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 408 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 409 show_one(scaling_min_freq, min);
 410 show_one(scaling_max_freq, max);
 411 show_one(scaling_cur_freq, cur);
 412
 413 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 414                                 struct cpufreq_policy *policy);
 415
 416 /**
 417  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 418  */
 419 #define store_one(file_name, object)                    \
 420 static ssize_t store_##file_name                                        \
 421 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 422 {                                                                       \
 423         unsigned int ret;                                               \
 424         struct cpufreq_policy new_policy;                               \
 425                                                                         \
 426         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 427         if (ret)                                                        \
 428                 return -EINVAL;                                         \
 429                                                                         \
 430         ret = sscanf(buf, "%u", &new_policy.object);                    \
 431         if (ret != 1)                                                   \
 432                 return -EINVAL;                                         \
 433                                                                         \
 434         ret = __cpufreq_set_policy(policy, &new_policy);                \
 435         policy->user_policy.object = policy->object;                    \
 436                                                                         \
 437         return ret ? ret : count;                                       \
 438 }
 439
 440 store_one(scaling_min_freq, min);
 441 store_one(scaling_max_freq, max);
 442
 443 /**
 444  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 445  */
 446 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 447                                         char *buf)
 448 {
 449         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 450         if (!cur_freq)
 451                 return sprintf(buf, "<unknown>");
 452         return sprintf(buf, "%u\n", cur_freq);
 453 }
 454
 455
 456 /**
 457  * show_scaling_governor - show the current policy for the specified CPU
 458  */
 459 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 460 {
 461         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 462                 return sprintf(buf, "powersave\n");
 463         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 464                 return sprintf(buf, "performance\n");
 465         else if (policy->governor)
 466                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 467                                 policy->governor->name);
 468         return -EINVAL;
 469 }
 470
 471
 472 /**
 473  * store_scaling_governor - store policy for the specified CPU
 474  */
 475 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 476                                         const char *buf, size_t count)
 477 {
 478         unsigned int ret;
 479         char    str_governor[16];
 480         struct cpufreq_policy new_policy;
 481
 482         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 483         if (ret)
 484                 return ret;
 485
 486         ret = sscanf(buf, "%15s", str_governor);
 487         if (ret != 1)
 488                 return -EINVAL;
 489
 490         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 491                                                 &new_policy.governor))
 492                 return -EINVAL;
 493
 494         /* Do not use cpufreq_set_policy here or the user_policy.max
 495            will be wrongly overridden */
 496         ret = __cpufreq_set_policy(policy, &new_policy);
 497
 498         policy->user_policy.policy = policy->policy;
 499         policy->user_policy.governor = policy->governor;
 500
 501         if (ret)
 502                 return ret;
 503         else
 504                 return count;
 505 }
 506
 507 /**
 508  * show_scaling_driver - show the cpufreq driver currently loaded
 509  */
 510 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 511 {
 512         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 513 }
 514
 515 /**
 516  * show_scaling_available_governors - show the available CPUfreq governors
 517  */
 518 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 519                                                 char *buf)
 520 {
 521         ssize_t i = 0;
 522         struct cpufreq_governor *t;
 523
 524         if (!cpufreq_driver->target) {
 525                 i += sprintf(buf, "performance powersave");
 526                 goto out;
 527         }
 528
 529         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 530                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 531                     - (CPUFREQ_NAME_LEN + 2)))
 532                         goto out;
 533                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 534         }
 535 out:
 536         i += sprintf(&buf[i], "\n");
 537         return i;
 538 }
 539
 540 static ssize_t show_cpus(const struct cpumask *mask, char *buf)
 541 {
 542         ssize_t i = 0;
 543         unsigned int cpu;
 544
 545         for_each_cpu(cpu, mask) {
 546                 if (i)
 547                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 548                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 549                 if (i >= (PAGE_SIZE - 5))
 550                         break;
 551         }
 552         i += sprintf(&buf[i], "\n");
 553         return i;
 554 }
 555
 556 /**
 557  * show_related_cpus - show the CPUs affected by each transition even if
 558  * hw coordination is in use
 559  */
 560 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 561 {
 562         return show_cpus(policy->related_cpus, buf);
 563 }
 564
 565 /**
 566  * show_affected_cpus - show the CPUs affected by each transition
 567  */
 568 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 569 {
 570         return show_cpus(policy->cpus, buf);
 571 }
 572
 573 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 574                                         const char *buf, size_t count)
 575 {
 576         unsigned int freq = 0;
 577         unsigned int ret;
 578
 579         if (!policy->governor || !policy->governor->store_setspeed)
 580                 return -EINVAL;
 581
 582         ret = sscanf(buf, "%u", &freq);
 583         if (ret != 1)
 584                 return -EINVAL;
 585
 586         policy->governor->store_setspeed(policy, freq);
 587
 588         return count;
 589 }
 590
 591 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 592 {
 593         if (!policy->governor || !policy->governor->show_setspeed)
 594                 return sprintf(buf, "<unsupported>\n");
 595
 596         return policy->governor->show_setspeed(policy, buf);
 597 }
 598
 599 /**
 600  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 601  */
 602 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 603 {
 604         unsigned int limit;
 605         int ret;
 606         if (cpufreq_driver->bios_limit) {
 607                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 608                 if (!ret)
 609                         return sprintf(buf, "%u\n", limit);
 610         }
 611         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 612 }
 613
 614 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 615 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 616 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 617 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 618 cpufreq_freq_attr_ro(scaling_available_governors);
 619 cpufreq_freq_attr_ro(scaling_driver);
 620 cpufreq_freq_attr_ro(scaling_cur_freq);
 621 cpufreq_freq_attr_ro(bios_limit);
 622 cpufreq_freq_attr_ro(related_cpus);
 623 cpufreq_freq_attr_ro(affected_cpus);
 624 cpufreq_freq_attr_rw(scaling_min_freq);
 625 cpufreq_freq_attr_rw(scaling_max_freq);
 626 cpufreq_freq_attr_rw(scaling_governor);
 627 cpufreq_freq_attr_rw(scaling_setspeed);
 628
 629 static struct attribute *default_attrs[] = {
 630         &cpuinfo_min_freq.attr,
 631         &cpuinfo_max_freq.attr,
 632         &cpuinfo_transition_latency.attr,
 633         &scaling_min_freq.attr,
 634         &scaling_max_freq.attr,
 635         &affected_cpus.attr,
 636         &related_cpus.attr,
 637         &scaling_governor.attr,
 638         &scaling_driver.attr,
 639         &scaling_available_governors.attr,
 640         &scaling_setspeed.attr,
 641         NULL
 642 };
 643
 644 struct kobject *cpufreq_global_kobject;
 645 EXPORT_SYMBOL(cpufreq_global_kobject);
 646
 647 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 648 #define to_attr(a) container_of(a, struct freq_attr, attr)
 649
 650 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 651 {
 652         struct cpufreq_policy *policy = to_policy(kobj);
 653         struct freq_attr *fattr = to_attr(attr);
 654         ssize_t ret = -EINVAL;
 655         policy = cpufreq_cpu_get_sysfs(policy->cpu);
 656         if (!policy)
 657                 goto no_policy;
 658
 659         if (lock_policy_rwsem_read(policy->cpu) < 0)
 660                 goto fail;
 661
 662         if (fattr->show)
 663                 ret = fattr->show(policy, buf);
 664         else
 665                 ret = -EIO;
 666
 667         unlock_policy_rwsem_read(policy->cpu);
 668 fail:
 669         cpufreq_cpu_put_sysfs(policy);
 670 no_policy:
 671         return ret;
 672 }
 673
 674 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 675                      const char *buf, size_t count)
 676 {
 677         struct cpufreq_policy *policy = to_policy(kobj);
 678         struct freq_attr *fattr = to_attr(attr);
 679         ssize_t ret = -EINVAL;
 680         policy = cpufreq_cpu_get_sysfs(policy->cpu);
 681         if (!policy)
 682                 goto no_policy;
 683
 684         if (lock_policy_rwsem_write(policy->cpu) < 0)
 685                 goto fail;
 686
 687         if (fattr->store)
 688                 ret = fattr->store(policy, buf, count);
 689         else
 690                 ret = -EIO;
 691
 692         unlock_policy_rwsem_write(policy->cpu);
 693 fail:
 694         cpufreq_cpu_put_sysfs(policy);
 695 no_policy:
 696         return ret;
 697 }
 698
 699 static void cpufreq_sysfs_release(struct kobject *kobj)
 700 {
 701         struct cpufreq_policy *policy = to_policy(kobj);
 702         pr_debug("last reference is dropped\n");
 703         complete(&policy->kobj_unregister);
 704 }
 705
 706 static const struct sysfs_ops sysfs_ops = {
 707         .show   = show,
 708         .store  = store,
 709 };
 710
 711 static struct kobj_type ktype_cpufreq = {
 712         .sysfs_ops      = &sysfs_ops,
 713         .default_attrs  = default_attrs,
 714         .release        = cpufreq_sysfs_release,
 715 };
 716
 717 /* symlink affected CPUs */
 718 static int cpufreq_add_dev_symlink(unsigned int cpu,
 719                                    struct cpufreq_policy *policy)
 720 {
 721         unsigned int j;
 722         int ret = 0;
 723
 724         for_each_cpu(j, policy->cpus) {
 725                 struct cpufreq_policy *managed_policy;
 726                 struct device *cpu_dev;
 727
 728                 if (j == cpu)
 729                         continue;
 730
 731                 pr_debug("CPU %u already managed, adding link\n", j);
 732                 managed_policy = cpufreq_cpu_get(cpu);
 733                 cpu_dev = get_cpu_device(j);
 734                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 735                                         "cpufreq");
 736                 if (ret) {
 737                         cpufreq_cpu_put(managed_policy);
 738                         return ret;
 739                 }
 740         }
 741         return ret;
 742 }
 743
 744 static int cpufreq_add_dev_interface(unsigned int cpu,
 745                                      struct cpufreq_policy *policy,
 746                                      struct device *dev)
 747 {
 748         struct cpufreq_policy new_policy;
 749         struct freq_attr **drv_attr;
 750         unsigned long flags;
 751         int ret = 0;
 752         unsigned int j;
 753
 754         /* prepare interface data */
 755         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 756                                    &dev->kobj, "cpufreq");
 757         if (ret)
 758                 return ret;
 759
 760         /* set up files for this cpu device */
 761         drv_attr = cpufreq_driver->attr;
 762         while ((drv_attr) && (*drv_attr)) {
 763                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 764                 if (ret)
 765                         goto err_out_kobj_put;
 766                 drv_attr++;
 767         }
 768         if (cpufreq_driver->get) {
 769                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 770                 if (ret)
 771                         goto err_out_kobj_put;
 772         }
 773         if (cpufreq_driver->target) {
 774                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 775                 if (ret)
 776                         goto err_out_kobj_put;
 777         }
 778         if (cpufreq_driver->bios_limit) {
 779                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 780                 if (ret)
 781                         goto err_out_kobj_put;
 782         }
 783
 784         write_lock_irqsave(&cpufreq_driver_lock, flags);
 785         for_each_cpu(j, policy->cpus) {
 786                 per_cpu(cpufreq_cpu_data, j) = policy;
 787                 per_cpu(cpufreq_policy_cpu, j) = policy->cpu;
 788         }
 789         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 790
 791         ret = cpufreq_add_dev_symlink(cpu, policy);
 792         if (ret)
 793                 goto err_out_kobj_put;
 794
 795         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 796         /* assure that the starting sequence is run in __cpufreq_set_policy */
 797         policy->governor = NULL;
 798
 799         /* set default policy */
 800         ret = __cpufreq_set_policy(policy, &new_policy);
 801         policy->user_policy.policy = policy->policy;
 802         policy->user_policy.governor = policy->governor;
 803
 804         if (ret) {
 805                 pr_debug("setting policy failed\n");
 806                 if (cpufreq_driver->exit)
 807                         cpufreq_driver->exit(policy);
 808         }
 809         return ret;
 810
 811 err_out_kobj_put:
 812         kobject_put(&policy->kobj);
 813         wait_for_completion(&policy->kobj_unregister);
 814         return ret;
 815 }
 816
 817 #ifdef CONFIG_HOTPLUG_CPU
 818 static int cpufreq_add_policy_cpu(unsigned int cpu, unsigned int sibling,
 819                                   struct device *dev)
 820 {
 821         struct cpufreq_policy *policy;
 822         int ret = 0, has_target = !!cpufreq_driver->target;
 823         unsigned long flags;
 824
 825         policy = cpufreq_cpu_get(sibling);
 826         WARN_ON(!policy);
 827
 828         if (has_target)
 829                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 830
 831         lock_policy_rwsem_write(sibling);
 832
 833         write_lock_irqsave(&cpufreq_driver_lock, flags);
 834
 835         cpumask_set_cpu(cpu, policy->cpus);
 836         per_cpu(cpufreq_policy_cpu, cpu) = policy->cpu;
 837         per_cpu(cpufreq_cpu_data, cpu) = policy;
 838         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 839
 840         unlock_policy_rwsem_write(sibling);
 841
 842         if (has_target) {
 843                 __cpufreq_governor(policy, CPUFREQ_GOV_START);
 844                 __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
 845         }
 846
 847         ret = sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
 848         if (ret) {
 849                 cpufreq_cpu_put(policy);
 850                 return ret;
 851         }
 852
 853         return 0;
 854 }
 855 #endif
 856
 857 /**
 858  * cpufreq_add_dev - add a CPU device
 859  *
 860  * Adds the cpufreq interface for a CPU device.
 861  *
 862  * The Oracle says: try running cpufreq registration/unregistration concurrently
 863  * with with cpu hotplugging and all hell will break loose. Tried to clean this
 864  * mess up, but more thorough testing is needed. - Mathieu
 865  */
 866 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
 867 {
 868         unsigned int j, cpu = dev->id;
 869         int ret = -ENOMEM;
 870         struct cpufreq_policy *policy;
 871         unsigned long flags;
 872 #ifdef CONFIG_HOTPLUG_CPU
 873         struct cpufreq_governor *gov;
 874         int sibling;
 875 #endif
 876
 877         if (cpu_is_offline(cpu))
 878                 return 0;
 879
 880         pr_debug("adding CPU %u\n", cpu);
 881
 882 #ifdef CONFIG_SMP
 883         /* check whether a different CPU already registered this
 884          * CPU because it is in the same boat. */
 885         policy = cpufreq_cpu_get(cpu);
 886         if (unlikely(policy)) {
 887                 cpufreq_cpu_put(policy);
 888                 return 0;
 889         }
 890
 891 #ifdef CONFIG_HOTPLUG_CPU
 892         /* Check if this cpu was hot-unplugged earlier and has siblings */
 893         read_lock_irqsave(&cpufreq_driver_lock, flags);
 894         for_each_online_cpu(sibling) {
 895                 struct cpufreq_policy *cp = per_cpu(cpufreq_cpu_data, sibling);
 896                 if (cp && cpumask_test_cpu(cpu, cp->related_cpus)) {
 897                         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 898                         return cpufreq_add_policy_cpu(cpu, sibling, dev);
 899                 }
 900         }
 901         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 902 #endif
 903 #endif
 904
 905         if (!try_module_get(cpufreq_driver->owner)) {
 906                 ret = -EINVAL;
 907                 goto module_out;
 908         }
 909
 910         policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
 911         if (!policy)
 912                 goto nomem_out;
 913
 914         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
 915                 goto err_free_policy;
 916
 917         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
 918                 goto err_free_cpumask;
 919
 920         policy->cpu = cpu;
 921         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
 922         cpumask_copy(policy->cpus, cpumask_of(cpu));
 923
 924         /* Initially set CPU itself as the policy_cpu */
 925         per_cpu(cpufreq_policy_cpu, cpu) = cpu;
 926
 927         init_completion(&policy->kobj_unregister);
 928         INIT_WORK(&policy->update, handle_update);
 929
 930         /* call driver. From then on the cpufreq must be able
 931          * to accept all calls to ->verify and ->setpolicy for this CPU
 932          */
 933         ret = cpufreq_driver->init(policy);
 934         if (ret) {
 935                 pr_debug("initialization failed\n");
 936                 goto err_set_policy_cpu;
 937         }
 938
 939         /* related cpus should atleast have policy->cpus */
 940         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
 941
 942         /*
 943          * affected cpus must always be the one, which are online. We aren't
 944          * managing offline cpus here.
 945          */
 946         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
 947
 948         policy->user_policy.min = policy->min;
 949         policy->user_policy.max = policy->max;
 950
 951         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
 952                                      CPUFREQ_START, policy);
 953
 954 #ifdef CONFIG_HOTPLUG_CPU
 955         gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
 956         if (gov) {
 957                 policy->governor = gov;
 958                 pr_debug("Restoring governor %s for cpu %d\n",
 959                        policy->governor->name, cpu);
 960         }
 961 #endif
 962
 963         ret = cpufreq_add_dev_interface(cpu, policy, dev);
 964         if (ret)
 965                 goto err_out_unregister;
 966
 967         kobject_uevent(&policy->kobj, KOBJ_ADD);
 968         module_put(cpufreq_driver->owner);
 969         pr_debug("initialization complete\n");
 970
 971         return 0;
 972
 973 err_out_unregister:
 974         write_lock_irqsave(&cpufreq_driver_lock, flags);
 975         for_each_cpu(j, policy->cpus)
 976                 per_cpu(cpufreq_cpu_data, j) = NULL;
 977         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 978
 979         kobject_put(&policy->kobj);
 980         wait_for_completion(&policy->kobj_unregister);
 981
 982 err_set_policy_cpu:
 983         per_cpu(cpufreq_policy_cpu, cpu) = -1;
 984         free_cpumask_var(policy->related_cpus);
 985 err_free_cpumask:
 986         free_cpumask_var(policy->cpus);
 987 err_free_policy:
 988         kfree(policy);
 989 nomem_out:
 990         module_put(cpufreq_driver->owner);
 991 module_out:
 992         return ret;
 993 }
 994
 995 static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
 996 {
 997         int j;
 998
 999         policy->last_cpu = policy->cpu;
1000         policy->cpu = cpu;
1001
1002         for_each_cpu(j, policy->cpus)
1003                 per_cpu(cpufreq_policy_cpu, j) = cpu;
1004
1005 #ifdef CONFIG_CPU_FREQ_TABLE
1006         cpufreq_frequency_table_update_policy_cpu(policy);
1007 #endif
1008         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1009                         CPUFREQ_UPDATE_POLICY_CPU, policy);
1010 }
1011
1012 /**
1013  * __cpufreq_remove_dev - remove a CPU device
1014  *
1015  * Removes the cpufreq interface for a CPU device.
1016  * Caller should already have policy_rwsem in write mode for this CPU.
1017  * This routine frees the rwsem before returning.
1018  */
1019 static int __cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1020 {
1021         unsigned int cpu = dev->id, ret, cpus;
1022         unsigned long flags;
1023         struct cpufreq_policy *data;
1024         struct kobject *kobj;
1025         struct completion *cmp;
1026         struct device *cpu_dev;
1027
1028         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1029
1030         write_lock_irqsave(&cpufreq_driver_lock, flags);
1031
1032         data = per_cpu(cpufreq_cpu_data, cpu);
1033         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1034
1035         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1036
1037         if (!data) {
1038                 pr_debug("%s: No cpu_data found\n", __func__);
1039                 return -EINVAL;
1040         }
1041
1042         if (cpufreq_driver->target)
1043                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1044
1045 #ifdef CONFIG_HOTPLUG_CPU
1046         if (!cpufreq_driver->setpolicy)
1047                 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1048                         data->governor->name, CPUFREQ_NAME_LEN);
1049 #endif
1050
1051         WARN_ON(lock_policy_rwsem_write(cpu));
1052         cpus = cpumask_weight(data->cpus);
1053
1054         if (cpus > 1)
1055                 cpumask_clear_cpu(cpu, data->cpus);
1056         unlock_policy_rwsem_write(cpu);
1057
1058         if (cpu != data->cpu) {
1059                 sysfs_remove_link(&dev->kobj, "cpufreq");
1060         } else if (cpus > 1) {
1061                 /* first sibling now owns the new sysfs dir */
1062                 cpu_dev = get_cpu_device(cpumask_first(data->cpus));
1063                 sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1064                 ret = kobject_move(&data->kobj, &cpu_dev->kobj);
1065                 if (ret) {
1066                         pr_err("%s: Failed to move kobj: %d", __func__, ret);
1067
1068                         WARN_ON(lock_policy_rwsem_write(cpu));
1069                         cpumask_set_cpu(cpu, data->cpus);
1070
1071                         write_lock_irqsave(&cpufreq_driver_lock, flags);
1072                         per_cpu(cpufreq_cpu_data, cpu) = data;
1073                         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1074
1075                         unlock_policy_rwsem_write(cpu);
1076
1077                         ret = sysfs_create_link(&cpu_dev->kobj, &data->kobj,
1078                                         "cpufreq");
1079                         return -EINVAL;
1080                 }
1081
1082                 WARN_ON(lock_policy_rwsem_write(cpu));
1083                 update_policy_cpu(data, cpu_dev->id);
1084                 unlock_policy_rwsem_write(cpu);
1085                 pr_debug("%s: policy Kobject moved to cpu: %d from: %d\n",
1086                                 __func__, cpu_dev->id, cpu);
1087         }
1088
1089         /* If cpu is last user of policy, free policy */
1090         if (cpus == 1) {
1091                 if (cpufreq_driver->target)
1092                         __cpufreq_governor(data, CPUFREQ_GOV_POLICY_EXIT);
1093
1094                 lock_policy_rwsem_read(cpu);
1095                 kobj = &data->kobj;
1096                 cmp = &data->kobj_unregister;
1097                 unlock_policy_rwsem_read(cpu);
1098                 kobject_put(kobj);
1099
1100                 /* we need to make sure that the underlying kobj is actually
1101                  * not referenced anymore by anybody before we proceed with
1102                  * unloading.
1103                  */
1104                 pr_debug("waiting for dropping of refcount\n");
1105                 wait_for_completion(cmp);
1106                 pr_debug("wait complete\n");
1107
1108                 if (cpufreq_driver->exit)
1109                         cpufreq_driver->exit(data);
1110
1111                 free_cpumask_var(data->related_cpus);
1112                 free_cpumask_var(data->cpus);
1113                 kfree(data);
1114         } else {
1115                 pr_debug("%s: removing link, cpu: %d\n", __func__, cpu);
1116                 cpufreq_cpu_put(data);
1117                 if (cpufreq_driver->target) {
1118                         __cpufreq_governor(data, CPUFREQ_GOV_START);
1119                         __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1120                 }
1121         }
1122
1123         per_cpu(cpufreq_policy_cpu, cpu) = -1;
1124         return 0;
1125 }
1126
1127
1128 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1129 {
1130         unsigned int cpu = dev->id;
1131         int retval;
1132
1133         if (cpu_is_offline(cpu))
1134                 return 0;
1135
1136         retval = __cpufreq_remove_dev(dev, sif);
1137         return retval;
1138 }
1139
1140
1141 static void handle_update(struct work_struct *work)
1142 {
1143         struct cpufreq_policy *policy =
1144                 container_of(work, struct cpufreq_policy, update);
1145         unsigned int cpu = policy->cpu;
1146         pr_debug("handle_update for cpu %u called\n", cpu);
1147         cpufreq_update_policy(cpu);
1148 }
1149
1150 /**
1151  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1152  *      @cpu: cpu number
1153  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1154  *      @new_freq: CPU frequency the CPU actually runs at
1155  *
1156  *      We adjust to current frequency first, and need to clean up later.
1157  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1158  */
1159 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1160                                 unsigned int new_freq)
1161 {
1162         struct cpufreq_policy *policy;
1163         struct cpufreq_freqs freqs;
1164         unsigned long flags;
1165
1166
1167         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing "
1168                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1169
1170         freqs.old = old_freq;
1171         freqs.new = new_freq;
1172
1173         read_lock_irqsave(&cpufreq_driver_lock, flags);
1174         policy = per_cpu(cpufreq_cpu_data, cpu);
1175         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1176
1177         cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE);
1178         cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE);
1179 }
1180
1181
1182 /**
1183  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1184  * @cpu: CPU number
1185  *
1186  * This is the last known freq, without actually getting it from the driver.
1187  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1188  */
1189 unsigned int cpufreq_quick_get(unsigned int cpu)
1190 {
1191         struct cpufreq_policy *policy;
1192         unsigned int ret_freq = 0;
1193
1194         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1195                 return cpufreq_driver->get(cpu);
1196
1197         policy = cpufreq_cpu_get(cpu);
1198         if (policy) {
1199                 ret_freq = policy->cur;
1200                 cpufreq_cpu_put(policy);
1201         }
1202
1203         return ret_freq;
1204 }
1205 EXPORT_SYMBOL(cpufreq_quick_get);
1206
1207 /**
1208  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1209  * @cpu: CPU number
1210  *
1211  * Just return the max possible frequency for a given CPU.
1212  */
1213 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1214 {
1215         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1216         unsigned int ret_freq = 0;
1217
1218         if (policy) {
1219                 ret_freq = policy->max;
1220                 cpufreq_cpu_put(policy);
1221         }
1222
1223         return ret_freq;
1224 }
1225 EXPORT_SYMBOL(cpufreq_quick_get_max);
1226
1227
1228 static unsigned int __cpufreq_get(unsigned int cpu)
1229 {
1230         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1231         unsigned int ret_freq = 0;
1232
1233         if (!cpufreq_driver->get)
1234                 return ret_freq;
1235
1236         ret_freq = cpufreq_driver->get(cpu);
1237
1238         if (ret_freq && policy->cur &&
1239                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1240                 /* verify no discrepancy between actual and
1241                                         saved value exists */
1242                 if (unlikely(ret_freq != policy->cur)) {
1243                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1244                         schedule_work(&policy->update);
1245                 }
1246         }
1247
1248         return ret_freq;
1249 }
1250
1251 /**
1252  * cpufreq_get - get the current CPU frequency (in kHz)
1253  * @cpu: CPU number
1254  *
1255  * Get the CPU current (static) CPU frequency
1256  */
1257 unsigned int cpufreq_get(unsigned int cpu)
1258 {
1259         unsigned int ret_freq = 0;
1260         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1261
1262         if (!policy)
1263                 goto out;
1264
1265         if (unlikely(lock_policy_rwsem_read(cpu)))
1266                 goto out_policy;
1267
1268         ret_freq = __cpufreq_get(cpu);
1269
1270         unlock_policy_rwsem_read(cpu);
1271
1272 out_policy:
1273         cpufreq_cpu_put(policy);
1274 out:
1275         return ret_freq;
1276 }
1277 EXPORT_SYMBOL(cpufreq_get);
1278
1279 static struct subsys_interface cpufreq_interface = {
1280         .name           = "cpufreq",
1281         .subsys         = &cpu_subsys,
1282         .add_dev        = cpufreq_add_dev,
1283         .remove_dev     = cpufreq_remove_dev,
1284 };
1285
1286
1287 /**
1288  * cpufreq_suspend() - Suspend CPUFreq governors
1289  *
1290  * Called during system wide Suspend/Hibernate cycles for suspending governors
1291  * as some platforms can't change frequency after this point in suspend cycle.
1292  * Because some of the devices (like: i2c, regulators, etc) they use for
1293  * changing frequency are suspended quickly after this point.
1294  */
1295 void cpufreq_suspend(void)
1296 {
1297         struct cpufreq_policy *policy;
1298         int cpu;
1299
1300         if (!cpufreq_driver)
1301                 return;
1302
1303         if (!has_target())
1304                 return;
1305
1306         pr_debug("%s: Suspending Governors\n", __func__);
1307
1308         for_each_possible_cpu(cpu) {
1309                 if (!cpu_online(cpu))
1310                         continue;
1311
1312                 policy = cpufreq_cpu_get(cpu);
1313
1314                 if (__cpufreq_governor(policy, CPUFREQ_GOV_STOP))
1315                         pr_err("%s: Failed to stop governor for policy: %p\n",
1316                                 __func__, policy);
1317                 else if (cpufreq_driver->suspend
1318                     && cpufreq_driver->suspend(policy))
1319                         pr_err("%s: Failed to suspend driver: %p\n", __func__,
1320                                 policy);
1321         }
1322
1323         cpufreq_suspended = true;
1324 }
1325
1326 /**
1327  * cpufreq_resume() - Resume CPUFreq governors
1328  *
1329  * Called during system wide Suspend/Hibernate cycle for resuming governors that
1330  * are suspended with cpufreq_suspend().
1331  */
1332 void cpufreq_resume(void)
1333 {
1334         struct cpufreq_policy *policy;
1335         int cpu;
1336
1337         if (!cpufreq_driver)
1338                 return;
1339
1340         if (!has_target())
1341                 return;
1342
1343         pr_debug("%s: Resuming Governors\n", __func__);
1344
1345         cpufreq_suspended = false;
1346
1347         for_each_possible_cpu(cpu) {
1348                 if (!cpu_online(cpu))
1349                         continue;
1350
1351                 policy = cpufreq_cpu_get(cpu);
1352
1353                 if (__cpufreq_governor(policy, CPUFREQ_GOV_START)
1354                     || __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))
1355                         pr_err("%s: Failed to start governor for policy: %p\n",
1356                                 __func__, policy);
1357                 else if (cpufreq_driver->resume
1358                     && cpufreq_driver->resume(policy))
1359                         pr_err("%s: Failed to resume driver: %p\n", __func__,
1360                                 policy);
1361
1362                 /*
1363                  * schedule call cpufreq_update_policy() for boot CPU, i.e. last
1364                  * policy in list. It will verify that the current freq is in
1365                  * sync with what we believe it to be.
1366                  */
1367                 if (cpu == 0)
1368                         schedule_work(&policy->update);
1369         }
1370 }
1371
1372 /**
1373  *      cpufreq_get_current_driver - return current driver's name
1374  *
1375  *      Return the name string of the currently loaded cpufreq driver
1376  *      or NULL, if none.
1377  */
1378 const char *cpufreq_get_current_driver(void)
1379 {
1380         if (cpufreq_driver)
1381                 return cpufreq_driver->name;
1382
1383         return NULL;
1384 }
1385 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1386
1387 /*********************************************************************
1388  *                     NOTIFIER LISTS INTERFACE                      *
1389  *********************************************************************/
1390
1391 /**
1392  *      cpufreq_register_notifier - register a driver with cpufreq
1393  *      @nb: notifier function to register
1394  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1395  *
1396  *      Add a driver to one of two lists: either a list of drivers that
1397  *      are notified about clock rate changes (once before and once after
1398  *      the transition), or a list of drivers that are notified about
1399  *      changes in cpufreq policy.
1400  *
1401  *      This function may sleep, and has the same return conditions as
1402  *      blocking_notifier_chain_register.
1403  */
1404 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1405 {
1406         int ret;
1407
1408         if (cpufreq_disabled())
1409                 return -EINVAL;
1410
1411         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1412
1413         switch (list) {
1414         case CPUFREQ_TRANSITION_NOTIFIER:
1415                 ret = srcu_notifier_chain_register(
1416                                 &cpufreq_transition_notifier_list, nb);
1417                 break;
1418         case CPUFREQ_POLICY_NOTIFIER:
1419                 ret = blocking_notifier_chain_register(
1420                                 &cpufreq_policy_notifier_list, nb);
1421                 break;
1422         default:
1423                 ret = -EINVAL;
1424         }
1425
1426         return ret;
1427 }
1428 EXPORT_SYMBOL(cpufreq_register_notifier);
1429
1430
1431 /**
1432  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1433  *      @nb: notifier block to be unregistered
1434  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1435  *
1436  *      Remove a driver from the CPU frequency notifier list.
1437  *
1438  *      This function may sleep, and has the same return conditions as
1439  *      blocking_notifier_chain_unregister.
1440  */
1441 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1442 {
1443         int ret;
1444
1445         if (cpufreq_disabled())
1446                 return -EINVAL;
1447
1448         switch (list) {
1449         case CPUFREQ_TRANSITION_NOTIFIER:
1450                 ret = srcu_notifier_chain_unregister(
1451                                 &cpufreq_transition_notifier_list, nb);
1452                 break;
1453         case CPUFREQ_POLICY_NOTIFIER:
1454                 ret = blocking_notifier_chain_unregister(
1455                                 &cpufreq_policy_notifier_list, nb);
1456                 break;
1457         default:
1458                 ret = -EINVAL;
1459         }
1460
1461         return ret;
1462 }
1463 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1464
1465
1466 /*********************************************************************
1467  *                              GOVERNORS                            *
1468  *********************************************************************/
1469
1470
1471 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1472                             unsigned int target_freq,
1473                             unsigned int relation)
1474 {
1475         int retval = -EINVAL;
1476         unsigned int old_target_freq = target_freq;
1477
1478         if (cpufreq_disabled())
1479                 return -ENODEV;
1480
1481         /* Make sure that target_freq is within supported range */
1482         if (target_freq > policy->max)
1483                 target_freq = policy->max;
1484         if (target_freq < policy->min)
1485                 target_freq = policy->min;
1486
1487         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1488                         policy->cpu, target_freq, relation, old_target_freq);
1489
1490         if (target_freq == policy->cur)
1491                 return 0;
1492
1493         if (cpufreq_driver->target)
1494                 retval = cpufreq_driver->target(policy, target_freq, relation);
1495
1496         return retval;
1497 }
1498 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1499
1500 int cpufreq_driver_target(struct cpufreq_policy *policy,
1501                           unsigned int target_freq,
1502                           unsigned int relation)
1503 {
1504         int ret = -EINVAL;
1505
1506         policy = cpufreq_cpu_get(policy->cpu);
1507         if (!policy)
1508                 goto no_policy;
1509
1510         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1511                 goto fail;
1512
1513         ret = __cpufreq_driver_target(policy, target_freq, relation);
1514
1515         unlock_policy_rwsem_write(policy->cpu);
1516
1517 fail:
1518         cpufreq_cpu_put(policy);
1519 no_policy:
1520         return ret;
1521 }
1522 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1523
1524 int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1525 {
1526         int ret = 0;
1527
1528         if (cpufreq_disabled())
1529                 return ret;
1530
1531         if (!cpufreq_driver->getavg)
1532                 return 0;
1533
1534         policy = cpufreq_cpu_get(policy->cpu);
1535         if (!policy)
1536                 return -EINVAL;
1537
1538         ret = cpufreq_driver->getavg(policy, cpu);
1539
1540         cpufreq_cpu_put(policy);
1541         return ret;
1542 }
1543 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1544
1545 /*
1546  * when "event" is CPUFREQ_GOV_LIMITS
1547  */
1548
1549 static int __cpufreq_governor(struct cpufreq_policy *policy,
1550                                         unsigned int event)
1551 {
1552         int ret;
1553
1554         /* Only must be defined when default governor is known to have latency
1555            restrictions, like e.g. conservative or ondemand.
1556            That this is the case is already ensured in Kconfig
1557         */
1558 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1559         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1560 #else
1561         struct cpufreq_governor *gov = NULL;
1562 #endif
1563
1564         /* Don't start any governor operations if we are entering suspend */
1565         if (cpufreq_suspended)
1566                 return 0;
1567
1568         if (policy->governor->max_transition_latency &&
1569             policy->cpuinfo.transition_latency >
1570             policy->governor->max_transition_latency) {
1571                 if (!gov)
1572                         return -EINVAL;
1573                 else {
1574                         printk(KERN_WARNING "%s governor failed, too long"
1575                                " transition latency of HW, fallback"
1576                                " to %s governor\n",
1577                                policy->governor->name,
1578                                gov->name);
1579                         policy->governor = gov;
1580                 }
1581         }
1582
1583         if (!try_module_get(policy->governor->owner))
1584                 return -EINVAL;
1585
1586         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1587                                                 policy->cpu, event);
1588
1589         mutex_lock(&cpufreq_governor_lock);
1590         if ((!policy->governor_enabled && (event == CPUFREQ_GOV_STOP)) ||
1591             (policy->governor_enabled && (event == CPUFREQ_GOV_START))) {
1592                 mutex_unlock(&cpufreq_governor_lock);
1593                 return -EBUSY;
1594         }
1595
1596         if (event == CPUFREQ_GOV_STOP)
1597                 policy->governor_enabled = false;
1598         else if (event == CPUFREQ_GOV_START)
1599                 policy->governor_enabled = true;
1600
1601         mutex_unlock(&cpufreq_governor_lock);
1602
1603         ret = policy->governor->governor(policy, event);
1604
1605         if (!ret) {
1606                 if (event == CPUFREQ_GOV_POLICY_INIT)
1607                         policy->governor->initialized++;
1608                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1609                         policy->governor->initialized--;
1610         } else {
1611                 /* Restore original values */
1612                 mutex_lock(&cpufreq_governor_lock);
1613                 if (event == CPUFREQ_GOV_STOP)
1614                         policy->governor_enabled = true;
1615                 else if (event == CPUFREQ_GOV_START)
1616                         policy->governor_enabled = false;
1617                 mutex_unlock(&cpufreq_governor_lock);
1618         }
1619
1620         /* we keep one module reference alive for
1621                         each CPU governed by this CPU */
1622         if ((event != CPUFREQ_GOV_START) || ret)
1623                 module_put(policy->governor->owner);
1624         if ((event == CPUFREQ_GOV_STOP) && !ret)
1625                 module_put(policy->governor->owner);
1626
1627         return ret;
1628 }
1629
1630
1631 int cpufreq_register_governor(struct cpufreq_governor *governor)
1632 {
1633         int err;
1634
1635         if (!governor)
1636                 return -EINVAL;
1637
1638         if (cpufreq_disabled())
1639                 return -ENODEV;
1640
1641         mutex_lock(&cpufreq_governor_mutex);
1642
1643         governor->initialized = 0;
1644         err = -EBUSY;
1645         if (__find_governor(governor->name) == NULL) {
1646                 err = 0;
1647                 list_add(&governor->governor_list, &cpufreq_governor_list);
1648         }
1649
1650         mutex_unlock(&cpufreq_governor_mutex);
1651         return err;
1652 }
1653 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1654
1655
1656 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1657 {
1658 #ifdef CONFIG_HOTPLUG_CPU
1659         int cpu;
1660 #endif
1661
1662         if (!governor)
1663                 return;
1664
1665         if (cpufreq_disabled())
1666                 return;
1667
1668 #ifdef CONFIG_HOTPLUG_CPU
1669         for_each_present_cpu(cpu) {
1670                 if (cpu_online(cpu))
1671                         continue;
1672                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
1673                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
1674         }
1675 #endif
1676
1677         mutex_lock(&cpufreq_governor_mutex);
1678         list_del(&governor->governor_list);
1679         mutex_unlock(&cpufreq_governor_mutex);
1680         return;
1681 }
1682 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1683
1684
1685
1686 /*********************************************************************
1687  *                          POLICY INTERFACE                         *
1688  *********************************************************************/
1689
1690 /**
1691  * cpufreq_get_policy - get the current cpufreq_policy
1692  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1693  *      is written
1694  *
1695  * Reads the current cpufreq policy.
1696  */
1697 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1698 {
1699         struct cpufreq_policy *cpu_policy;
1700         if (!policy)
1701                 return -EINVAL;
1702
1703         cpu_policy = cpufreq_cpu_get(cpu);
1704         if (!cpu_policy)
1705                 return -EINVAL;
1706
1707         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1708
1709         cpufreq_cpu_put(cpu_policy);
1710         return 0;
1711 }
1712 EXPORT_SYMBOL(cpufreq_get_policy);
1713
1714
1715 /*
1716  * data   : current policy.
1717  * policy : policy to be set.
1718  */
1719 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1720                                 struct cpufreq_policy *policy)
1721 {
1722         int ret = 0, failed = 1;
1723
1724         pr_debug("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1725                 policy->min, policy->max);
1726
1727         memcpy(&policy->cpuinfo, &data->cpuinfo,
1728                                 sizeof(struct cpufreq_cpuinfo));
1729
1730         if (policy->min > data->max || policy->max < data->min) {
1731                 ret = -EINVAL;
1732                 goto error_out;
1733         }
1734
1735         /* verify the cpu speed can be set within this limit */
1736         ret = cpufreq_driver->verify(policy);
1737         if (ret)
1738                 goto error_out;
1739
1740         /* adjust if necessary - all reasons */
1741         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1742                         CPUFREQ_ADJUST, policy);
1743
1744         /* adjust if necessary - hardware incompatibility*/
1745         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1746                         CPUFREQ_INCOMPATIBLE, policy);
1747
1748         /* verify the cpu speed can be set within this limit,
1749            which might be different to the first one */
1750         ret = cpufreq_driver->verify(policy);
1751         if (ret)
1752                 goto error_out;
1753
1754         /* notification of the new policy */
1755         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1756                         CPUFREQ_NOTIFY, policy);
1757
1758         data->min = policy->min;
1759         data->max = policy->max;
1760
1761         pr_debug("new min and max freqs are %u - %u kHz\n",
1762                                         data->min, data->max);
1763
1764         if (cpufreq_driver->setpolicy) {
1765                 data->policy = policy->policy;
1766                 pr_debug("setting range\n");
1767                 ret = cpufreq_driver->setpolicy(policy);
1768         } else {
1769                 if (policy->governor != data->governor) {
1770                         /* save old, working values */
1771                         struct cpufreq_governor *old_gov = data->governor;
1772
1773                         pr_debug("governor switch\n");
1774
1775                         /* end old governor */
1776                         if (data->governor) {
1777                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1778                                 unlock_policy_rwsem_write(policy->cpu);
1779                                 __cpufreq_governor(data,
1780                                                 CPUFREQ_GOV_POLICY_EXIT);
1781                                 lock_policy_rwsem_write(policy->cpu);
1782                         }
1783
1784                         /* start new governor */
1785                         data->governor = policy->governor;
1786                         if (!__cpufreq_governor(data, CPUFREQ_GOV_POLICY_INIT)) {
1787                                 if (!__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1788                                         failed = 0;
1789                                 } else {
1790                                         unlock_policy_rwsem_write(policy->cpu);
1791                                         __cpufreq_governor(data,
1792                                                         CPUFREQ_GOV_POLICY_EXIT);
1793                                         lock_policy_rwsem_write(policy->cpu);
1794                                 }
1795                         }
1796
1797                         if (failed) {
1798                                 /* new governor failed, so re-start old one */
1799                                 pr_debug("starting governor %s failed\n",
1800                                                         data->governor->name);
1801                                 if (old_gov) {
1802                                         data->governor = old_gov;
1803                                         __cpufreq_governor(data,
1804                                                         CPUFREQ_GOV_POLICY_INIT);
1805                                         __cpufreq_governor(data,
1806                                                            CPUFREQ_GOV_START);
1807                                 }
1808                                 ret = -EINVAL;
1809                                 goto error_out;
1810                         }
1811                         /* might be a policy change, too, so fall through */
1812                 }
1813                 pr_debug("governor: change or update limits\n");
1814                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1815         }
1816
1817 error_out:
1818         return ret;
1819 }
1820
1821 /**
1822  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1823  *      @cpu: CPU which shall be re-evaluated
1824  *
1825  *      Useful for policy notifiers which have different necessities
1826  *      at different times.
1827  */
1828 int cpufreq_update_policy(unsigned int cpu)
1829 {
1830         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1831         struct cpufreq_policy policy;
1832         int ret;
1833
1834         if (!data) {
1835                 ret = -ENODEV;
1836                 goto no_policy;
1837         }
1838
1839         if (unlikely(lock_policy_rwsem_write(cpu))) {
1840                 ret = -EINVAL;
1841                 goto fail;
1842         }
1843
1844         pr_debug("updating policy for CPU %u\n", cpu);
1845         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1846         policy.min = data->user_policy.min;
1847         policy.max = data->user_policy.max;
1848         policy.policy = data->user_policy.policy;
1849         policy.governor = data->user_policy.governor;
1850
1851         /* BIOS might change freq behind our back
1852           -> ask driver for current freq and notify governors about a change */
1853         if (cpufreq_driver->get) {
1854                 policy.cur = cpufreq_driver->get(cpu);
1855                 if (!data->cur) {
1856                         pr_debug("Driver did not initialize current freq");
1857                         data->cur = policy.cur;
1858                 } else {
1859                         if (data->cur != policy.cur && cpufreq_driver->target)
1860                                 cpufreq_out_of_sync(cpu, data->cur,
1861                                                                 policy.cur);
1862                 }
1863         }
1864
1865         ret = __cpufreq_set_policy(data, &policy);
1866
1867         unlock_policy_rwsem_write(cpu);
1868
1869 fail:
1870         cpufreq_cpu_put(data);
1871 no_policy:
1872         return ret;
1873 }
1874 EXPORT_SYMBOL(cpufreq_update_policy);
1875
1876 static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1877                                         unsigned long action, void *hcpu)
1878 {
1879         unsigned int cpu = (unsigned long)hcpu;
1880         struct device *dev;
1881
1882         dev = get_cpu_device(cpu);
1883         if (dev) {
1884                 switch (action) {
1885                 case CPU_ONLINE:
1886                 case CPU_ONLINE_FROZEN:
1887                         cpufreq_add_dev(dev, NULL);
1888                         break;
1889                 case CPU_DOWN_PREPARE:
1890                 case CPU_DOWN_PREPARE_FROZEN:
1891                         __cpufreq_remove_dev(dev, NULL);
1892                         break;
1893                 case CPU_DOWN_FAILED:
1894                 case CPU_DOWN_FAILED_FROZEN:
1895                         cpufreq_add_dev(dev, NULL);
1896                         break;
1897                 }
1898         }
1899         return NOTIFY_OK;
1900 }
1901
1902 static struct notifier_block __refdata cpufreq_cpu_notifier = {
1903     .notifier_call = cpufreq_cpu_callback,
1904 };
1905
1906 /*********************************************************************
1907  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1908  *********************************************************************/
1909
1910 /**
1911  * cpufreq_register_driver - register a CPU Frequency driver
1912  * @driver_data: A struct cpufreq_driver containing the values#
1913  * submitted by the CPU Frequency driver.
1914  *
1915  *   Registers a CPU Frequency driver to this core code. This code
1916  * returns zero on success, -EBUSY when another driver got here first
1917  * (and isn't unregistered in the meantime).
1918  *
1919  */
1920 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1921 {
1922         unsigned long flags;
1923         int ret;
1924
1925         if (cpufreq_disabled())
1926                 return -ENODEV;
1927
1928         if (!driver_data || !driver_data->verify || !driver_data->init ||
1929             ((!driver_data->setpolicy) && (!driver_data->target)))
1930                 return -EINVAL;
1931
1932         pr_debug("trying to register driver %s\n", driver_data->name);
1933
1934         if (driver_data->setpolicy)
1935                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1936
1937         write_lock_irqsave(&cpufreq_driver_lock, flags);
1938         if (cpufreq_driver) {
1939                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1940                 return -EBUSY;
1941         }
1942         cpufreq_driver = driver_data;
1943         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1944
1945         ret = subsys_interface_register(&cpufreq_interface);
1946         if (ret)
1947                 goto err_null_driver;
1948
1949         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
1950                 int i;
1951                 ret = -ENODEV;
1952
1953                 /* check for at least one working CPU */
1954                 for (i = 0; i < nr_cpu_ids; i++)
1955                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
1956                                 ret = 0;
1957                                 break;
1958                         }
1959
1960                 /* if all ->init() calls failed, unregister */
1961                 if (ret) {
1962                         pr_debug("no CPU initialized for driver %s\n",
1963                                                         driver_data->name);
1964                         goto err_if_unreg;
1965                 }
1966         }
1967
1968         register_hotcpu_notifier(&cpufreq_cpu_notifier);
1969         pr_debug("driver %s up and running\n", driver_data->name);
1970
1971         return 0;
1972 err_if_unreg:
1973         subsys_interface_unregister(&cpufreq_interface);
1974 err_null_driver:
1975         write_lock_irqsave(&cpufreq_driver_lock, flags);
1976         cpufreq_driver = NULL;
1977         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1978         return ret;
1979 }
1980 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
1981
1982
1983 /**
1984  * cpufreq_unregister_driver - unregister the current CPUFreq driver
1985  *
1986  *    Unregister the current CPUFreq driver. Only call this if you have
1987  * the right to do so, i.e. if you have succeeded in initialising before!
1988  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
1989  * currently not initialised.
1990  */
1991 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
1992 {
1993         unsigned long flags;
1994
1995         if (!cpufreq_driver || (driver != cpufreq_driver))
1996                 return -EINVAL;
1997
1998         pr_debug("unregistering driver %s\n", driver->name);
1999
2000         subsys_interface_unregister(&cpufreq_interface);
2001         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2002
2003         write_lock_irqsave(&cpufreq_driver_lock, flags);
2004         cpufreq_driver = NULL;
2005         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2006
2007         return 0;
2008 }
2009 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2010
2011 static int __init cpufreq_core_init(void)
2012 {
2013         int cpu;
2014
2015         if (cpufreq_disabled())
2016                 return -ENODEV;
2017
2018         for_each_possible_cpu(cpu) {
2019                 per_cpu(cpufreq_policy_cpu, cpu) = -1;
2020                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
2021         }
2022
2023         cpufreq_global_kobject = kobject_create_and_add("cpufreq", &cpu_subsys.dev_root->kobj);
2024         BUG_ON(!cpufreq_global_kobject);
2025
2026         return 0;
2027 }
2028 core_initcall(cpufreq_core_init);