drivers/cpufreq/arm_big_little.c

   1 /*
   2  * ARM big.LITTLE Platforms CPUFreq support
   3  *
   4  * Copyright (C) 2013 ARM Ltd.
   5  * Sudeep KarkadaNagesha <sudeep.karkadanagesha@arm.com>
   6  *
   7  * Copyright (C) 2013 Linaro.
   8  * Viresh Kumar <viresh.kumar@linaro.org>
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License version 2 as
  12  * published by the Free Software Foundation.
  13  *
  14  * This program is distributed "as is" WITHOUT ANY WARRANTY of any
  15  * kind, whether express or implied; without even the implied warranty
  16  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17  * GNU General Public License for more details.
  18  */
  19
  20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  21
  22 #include <linux/clk.h>
  23 #include <linux/cpu.h>
  24 #include <linux/cpufreq.h>
  25 #include <linux/cpumask.h>
  26 #include <linux/export.h>
  27 #include <linux/mutex.h>
  28 #include <linux/of_platform.h>
  29 #include <linux/opp.h>
  30 #include <linux/slab.h>
  31 #include <linux/topology.h>
  32 #include <linux/types.h>
  33 #include <asm/bL_switcher.h>
  34
  35 #include "arm_big_little.h"
  36
  37 #ifdef CONFIG_BL_SWITCHER
  38 bool bL_switching_enabled;
  39 #endif
  40
  41 #define ACTUAL_FREQ(cluster, freq)      ((cluster == A7_CLUSTER) ? freq << 1 : freq)
  42 #define VIRT_FREQ(cluster, freq)        ((cluster == A7_CLUSTER) ? freq >> 1 : freq)
  43
  44 static struct cpufreq_arm_bL_ops *arm_bL_ops;
  45 static struct clk *clk[MAX_CLUSTERS];
  46 static struct cpufreq_frequency_table *freq_table[MAX_CLUSTERS + 1];
  47 static atomic_t cluster_usage[MAX_CLUSTERS + 1] = {ATOMIC_INIT(0),
  48         ATOMIC_INIT(0)};
  49
  50 static unsigned int clk_big_min;        /* (Big) clock frequencies */
  51 static unsigned int clk_little_max;     /* Maximum clock frequency (Little) */
  52
  53 static DEFINE_PER_CPU(unsigned int, physical_cluster);
  54 static DEFINE_PER_CPU(unsigned int, cpu_last_req_freq);
  55
  56 static struct mutex cluster_lock[MAX_CLUSTERS];
  57
  58 static unsigned int find_cluster_maxfreq(int cluster)
  59 {
  60         int j;
  61         u32 max_freq = 0, cpu_freq;
  62
  63         for_each_online_cpu(j) {
  64                 cpu_freq = per_cpu(cpu_last_req_freq, j);
  65
  66                 if ((cluster == per_cpu(physical_cluster, j)) &&
  67                                 (max_freq < cpu_freq))
  68                         max_freq = cpu_freq;
  69         }
  70
  71         pr_debug("%s: cluster: %d, max freq: %d\n", __func__, cluster,
  72                         max_freq);
  73
  74         return max_freq;
  75 }
  76
  77 static unsigned int clk_get_cpu_rate(unsigned int cpu)
  78 {
  79         u32 cur_cluster = per_cpu(physical_cluster, cpu);
  80         u32 rate = clk_get_rate(clk[cur_cluster]) / 1000;
  81
  82         /* For switcher we use virtual A15 clock rates */
  83         if (is_bL_switching_enabled())
  84                 rate = VIRT_FREQ(cur_cluster, rate);
  85
  86         pr_debug("%s: cpu: %d, cluster: %d, freq: %u\n", __func__, cpu,
  87                         cur_cluster, rate);
  88
  89         return rate;
  90 }
  91
  92 static unsigned int bL_cpufreq_get_rate(unsigned int cpu)
  93 {
  94         if (is_bL_switching_enabled()) {
  95                 pr_debug("%s: freq: %d\n", __func__, per_cpu(cpu_last_req_freq,
  96                                         cpu));
  97
  98                 return per_cpu(cpu_last_req_freq, cpu);
  99         } else {
 100                 return clk_get_cpu_rate(cpu);
 101         }
 102 }
 103
 104 static unsigned int
 105 bL_cpufreq_set_rate(u32 cpu, u32 old_cluster, u32 new_cluster, u32 rate)
 106 {
 107         u32 new_rate, prev_rate;
 108         int ret;
 109         bool bLs = is_bL_switching_enabled();
 110
 111         mutex_lock(&cluster_lock[new_cluster]);
 112
 113         if (bLs) {
 114                 prev_rate = per_cpu(cpu_last_req_freq, cpu);
 115                 per_cpu(cpu_last_req_freq, cpu) = rate;
 116                 per_cpu(physical_cluster, cpu) = new_cluster;
 117
 118                 new_rate = find_cluster_maxfreq(new_cluster);
 119                 new_rate = ACTUAL_FREQ(new_cluster, new_rate);
 120         } else {
 121                 new_rate = rate;
 122         }
 123
 124         pr_debug("%s: cpu: %d, old cluster: %d, new cluster: %d, freq: %d\n",
 125                         __func__, cpu, old_cluster, new_cluster, new_rate);
 126
 127         ret = clk_set_rate(clk[new_cluster], new_rate * 1000);
 128         if (WARN_ON(ret)) {
 129                 pr_err("clk_set_rate failed: %d, new cluster: %d\n", ret,
 130                                 new_cluster);
 131                 if (bLs) {
 132                         per_cpu(cpu_last_req_freq, cpu) = prev_rate;
 133                         per_cpu(physical_cluster, cpu) = old_cluster;
 134                 }
 135
 136                 mutex_unlock(&cluster_lock[new_cluster]);
 137
 138                 return ret;
 139         }
 140
 141         mutex_unlock(&cluster_lock[new_cluster]);
 142
 143         /* Recalc freq for old cluster when switching clusters */
 144         if (old_cluster != new_cluster) {
 145                 pr_debug("%s: cpu: %d, old cluster: %d, new cluster: %d\n",
 146                                 __func__, cpu, old_cluster, new_cluster);
 147
 148                 /* Switch cluster */
 149                 bL_switch_request(cpu, new_cluster);
 150
 151                 mutex_lock(&cluster_lock[old_cluster]);
 152
 153                 /* Set freq of old cluster if there are cpus left on it */
 154                 new_rate = find_cluster_maxfreq(old_cluster);
 155                 new_rate = ACTUAL_FREQ(old_cluster, new_rate);
 156
 157                 if (new_rate) {
 158                         pr_debug("%s: Updating rate of old cluster: %d, to freq: %d\n",
 159                                         __func__, old_cluster, new_rate);
 160
 161                         if (clk_set_rate(clk[old_cluster], new_rate * 1000))
 162                                 pr_err("%s: clk_set_rate failed: %d, old cluster: %d\n",
 163                                                 __func__, ret, old_cluster);
 164                 }
 165                 mutex_unlock(&cluster_lock[old_cluster]);
 166         }
 167
 168         return 0;
 169 }
 170
 171 /* Validate policy frequency range */
 172 static int bL_cpufreq_verify_policy(struct cpufreq_policy *policy)
 173 {
 174         u32 cur_cluster = cpu_to_cluster(policy->cpu);
 175
 176         return cpufreq_frequency_table_verify(policy, freq_table[cur_cluster]);
 177 }
 178
 179 /* Set clock frequency */
 180 static int bL_cpufreq_set_target(struct cpufreq_policy *policy,
 181                 unsigned int target_freq, unsigned int relation)
 182 {
 183         struct cpufreq_freqs freqs;
 184         u32 cpu = policy->cpu, freq_tab_idx, cur_cluster, new_cluster,
 185             actual_cluster;
 186         int ret = 0;
 187
 188         cur_cluster = cpu_to_cluster(cpu);
 189         new_cluster = actual_cluster = per_cpu(physical_cluster, cpu);
 190
 191         freqs.old = bL_cpufreq_get_rate(cpu);
 192
 193         /* Determine valid target frequency using freq_table */
 194         cpufreq_frequency_table_target(policy, freq_table[cur_cluster],
 195                         target_freq, relation, &freq_tab_idx);
 196         freqs.new = freq_table[cur_cluster][freq_tab_idx].frequency;
 197
 198         pr_debug("%s: cpu: %d, cluster: %d, oldfreq: %d, target freq: %d, new freq: %d\n",
 199                         __func__, cpu, cur_cluster, freqs.old, target_freq,
 200                         freqs.new);
 201
 202         if (freqs.old == freqs.new)
 203                 return 0;
 204
 205         if (is_bL_switching_enabled()) {
 206                 if ((actual_cluster == A15_CLUSTER) &&
 207                                 (freqs.new < clk_big_min)) {
 208                         new_cluster = A7_CLUSTER;
 209                 } else if ((actual_cluster == A7_CLUSTER) &&
 210                                 (freqs.new > clk_little_max)) {
 211                         new_cluster = A15_CLUSTER;
 212                 }
 213         }
 214
 215         cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE);
 216
 217         ret = bL_cpufreq_set_rate(cpu, actual_cluster, new_cluster, freqs.new);
 218         if (ret)
 219                 return ret;
 220
 221         policy->cur = freqs.new;
 222
 223         cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE);
 224
 225         return ret;
 226 }
 227
 228 static inline u32 get_table_count(struct cpufreq_frequency_table *table)
 229 {
 230         int count;
 231
 232         for (count = 0; table[count].frequency != CPUFREQ_TABLE_END; count++)
 233                 ;
 234
 235         return count;
 236 }
 237
 238 /* get the minimum frequency in the cpufreq_frequency_table */
 239 static inline u32 get_table_min(struct cpufreq_frequency_table *table)
 240 {
 241         int i;
 242         uint32_t min_freq = ~0;
 243         for (i = 0; (table[i].frequency != CPUFREQ_TABLE_END); i++)
 244                 if (table[i].frequency < min_freq)
 245                         min_freq = table[i].frequency;
 246         return min_freq;
 247 }
 248
 249 /* get the maximum frequency in the cpufreq_frequency_table */
 250 static inline u32 get_table_max(struct cpufreq_frequency_table *table)
 251 {
 252         int i;
 253         uint32_t max_freq = 0;
 254         for (i = 0; (table[i].frequency != CPUFREQ_TABLE_END); i++)
 255                 if (table[i].frequency > max_freq)
 256                         max_freq = table[i].frequency;
 257         return max_freq;
 258 }
 259
 260 static int merge_cluster_tables(void)
 261 {
 262         int i, j, k = 0, count = 1;
 263         struct cpufreq_frequency_table *table;
 264
 265         for (i = 0; i < MAX_CLUSTERS; i++)
 266                 count += get_table_count(freq_table[i]);
 267
 268         table = kzalloc(sizeof(*table) * count, GFP_KERNEL);
 269         if (!table)
 270                 return -ENOMEM;
 271
 272         freq_table[MAX_CLUSTERS] = table;
 273
 274         /* Add in reverse order to get freqs in increasing order */
 275         for (i = MAX_CLUSTERS - 1; i >= 0; i--) {
 276                 for (j = 0; freq_table[i][j].frequency != CPUFREQ_TABLE_END;
 277                                 j++) {
 278                         table[k].frequency = VIRT_FREQ(i,
 279                                         freq_table[i][j].frequency);
 280                         pr_debug("%s: index: %d, freq: %d\n", __func__, k,
 281                                         table[k].frequency);
 282                         k++;
 283                 }
 284         }
 285
 286         table[k].index = k;
 287         table[k].frequency = CPUFREQ_TABLE_END;
 288
 289         pr_debug("%s: End, table: %p, count: %d\n", __func__, table, k);
 290
 291         return 0;
 292 }
 293
 294 static void _put_cluster_clk_and_freq_table(struct device *cpu_dev)
 295 {
 296         u32 cluster = cpu_to_cluster(cpu_dev->id);
 297
 298         if (!atomic_dec_return(&cluster_usage[cluster])) {
 299                 clk_put(clk[cluster]);
 300                 opp_free_cpufreq_table(cpu_dev, &freq_table[cluster]);
 301                 dev_dbg(cpu_dev, "%s: cluster: %d\n", __func__, cluster);
 302         }
 303 }
 304
 305 static void put_cluster_clk_and_freq_table(struct device *cpu_dev)
 306 {
 307         u32 cluster = cpu_to_cluster(cpu_dev->id);
 308         int i;
 309
 310         if (cluster < MAX_CLUSTERS)
 311                 return _put_cluster_clk_and_freq_table(cpu_dev);
 312
 313         if (atomic_dec_return(&cluster_usage[MAX_CLUSTERS]))
 314                 return;
 315
 316         for (i = 0; i < MAX_CLUSTERS; i++) {
 317                 struct device *cdev = get_cpu_device(i);
 318                 if (!cdev) {
 319                         pr_err("%s: failed to get cpu%d device\n", __func__, i);
 320                         return;
 321                 }
 322
 323                 _put_cluster_clk_and_freq_table(cdev);
 324         }
 325
 326         /* free virtual table */
 327         kfree(freq_table[MAX_CLUSTERS]);
 328 }
 329
 330 static int _get_cluster_clk_and_freq_table(struct device *cpu_dev)
 331 {
 332         u32 cluster = cpu_to_cluster(cpu_dev->id);
 333         char name[14] = "cpu-cluster.X";
 334         int ret;
 335
 336         if (atomic_inc_return(&cluster_usage[cluster]) != 1)
 337                 return 0;
 338
 339         ret = arm_bL_ops->init_opp_table(cpu_dev);
 340         if (ret) {
 341                 dev_err(cpu_dev, "%s: init_opp_table failed, cpu: %d, err: %d\n",
 342                                 __func__, cpu_dev->id, ret);
 343                 goto atomic_dec;
 344         }
 345
 346         ret = opp_init_cpufreq_table(cpu_dev, &freq_table[cluster]);
 347         if (ret) {
 348                 dev_err(cpu_dev, "%s: failed to init cpufreq table, cpu: %d, err: %d\n",
 349                                 __func__, cpu_dev->id, ret);
 350                 goto atomic_dec;
 351         }
 352
 353         name[12] = cluster + '0';
 354         clk[cluster] = clk_get_sys(name, NULL);
 355         if (!IS_ERR(clk[cluster])) {
 356                 dev_dbg(cpu_dev, "%s: clk: %p & freq table: %p, cluster: %d\n",
 357                                 __func__, clk[cluster], freq_table[cluster],
 358                                 cluster);
 359                 return 0;
 360         }
 361
 362         dev_err(cpu_dev, "%s: Failed to get clk for cpu: %d, cluster: %d\n",
 363                         __func__, cpu_dev->id, cluster);
 364         ret = PTR_ERR(clk[cluster]);
 365         opp_free_cpufreq_table(cpu_dev, &freq_table[cluster]);
 366
 367 atomic_dec:
 368         atomic_dec(&cluster_usage[cluster]);
 369         dev_err(cpu_dev, "%s: Failed to get data for cluster: %d\n", __func__,
 370                         cluster);
 371         return ret;
 372 }
 373
 374 static int get_cluster_clk_and_freq_table(struct device *cpu_dev)
 375 {
 376         u32 cluster = cpu_to_cluster(cpu_dev->id);
 377         int i, ret;
 378
 379         if (cluster < MAX_CLUSTERS)
 380                 return _get_cluster_clk_and_freq_table(cpu_dev);
 381
 382         if (atomic_inc_return(&cluster_usage[MAX_CLUSTERS]) != 1)
 383                 return 0;
 384
 385         /*
 386          * Get data for all clusters and fill virtual cluster with a merge of
 387          * both
 388          */
 389         for (i = 0; i < MAX_CLUSTERS; i++) {
 390                 struct device *cdev = get_cpu_device(i);
 391                 if (!cdev) {
 392                         pr_err("%s: failed to get cpu%d device\n", __func__, i);
 393                         return -ENODEV;
 394                 }
 395
 396                 ret = _get_cluster_clk_and_freq_table(cdev);
 397                 if (ret)
 398                         goto put_clusters;
 399         }
 400
 401         ret = merge_cluster_tables();
 402         if (ret)
 403                 goto put_clusters;
 404
 405         /* Assuming 2 cluster, set clk_big_min and clk_little_max */
 406         clk_big_min = get_table_min(freq_table[0]);
 407         clk_little_max = VIRT_FREQ(1, get_table_max(freq_table[1]));
 408
 409         pr_debug("%s: cluster: %d, clk_big_min: %d, clk_little_max: %d\n",
 410                         __func__, cluster, clk_big_min, clk_little_max);
 411
 412         return 0;
 413
 414 put_clusters:
 415         while (i--) {
 416                 struct device *cdev = get_cpu_device(i);
 417                 if (!cdev) {
 418                         pr_err("%s: failed to get cpu%d device\n", __func__, i);
 419                         return -ENODEV;
 420                 }
 421
 422                 _put_cluster_clk_and_freq_table(cdev);
 423         }
 424
 425         atomic_dec(&cluster_usage[MAX_CLUSTERS]);
 426
 427         return ret;
 428 }
 429
 430 /* Per-CPU initialization */
 431 static int bL_cpufreq_init(struct cpufreq_policy *policy)
 432 {
 433         u32 cur_cluster = cpu_to_cluster(policy->cpu);
 434         struct device *cpu_dev;
 435         int ret;
 436
 437         cpu_dev = get_cpu_device(policy->cpu);
 438         if (!cpu_dev) {
 439                 pr_err("%s: failed to get cpu%d device\n", __func__,
 440                                 policy->cpu);
 441                 return -ENODEV;
 442         }
 443
 444         ret = get_cluster_clk_and_freq_table(cpu_dev);
 445         if (ret)
 446                 return ret;
 447
 448         ret = cpufreq_frequency_table_cpuinfo(policy, freq_table[cur_cluster]);
 449         if (ret) {
 450                 dev_err(cpu_dev, "CPU %d, cluster: %d invalid freq table\n",
 451                                 policy->cpu, cur_cluster);
 452                 put_cluster_clk_and_freq_table(cpu_dev);
 453                 return ret;
 454         }
 455
 456         cpufreq_frequency_table_get_attr(freq_table[cur_cluster], policy->cpu);
 457
 458         if (cur_cluster < MAX_CLUSTERS) {
 459                 int cpu;
 460
 461                 cpumask_copy(policy->cpus, topology_core_cpumask(policy->cpu));
 462
 463                 for_each_cpu(cpu, policy->cpus)
 464                         per_cpu(physical_cluster, cpu) = cur_cluster;
 465         } else {
 466                 /* Assumption: during init, we are always running on A15 */
 467                 per_cpu(physical_cluster, policy->cpu) = A15_CLUSTER;
 468         }
 469
 470         if (arm_bL_ops->get_transition_latency)
 471                 policy->cpuinfo.transition_latency =
 472                         arm_bL_ops->get_transition_latency(cpu_dev);
 473         else
 474                 policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
 475
 476         policy->cur = clk_get_cpu_rate(policy->cpu);
 477
 478         if (is_bL_switching_enabled())
 479                 per_cpu(cpu_last_req_freq, policy->cpu) = policy->cur;
 480
 481         dev_info(cpu_dev, "%s: CPU %d initialized\n", __func__, policy->cpu);
 482         return 0;
 483 }
 484
 485 /* Export freq_table to sysfs */
 486 static struct freq_attr *bL_cpufreq_attr[] = {
 487         &cpufreq_freq_attr_scaling_available_freqs,
 488         NULL,
 489 };
 490
 491 static struct cpufreq_driver bL_cpufreq_driver = {
 492         .name                   = "arm-big-little",
 493         .flags                  = CPUFREQ_STICKY,
 494         .verify                 = bL_cpufreq_verify_policy,
 495         .target                 = bL_cpufreq_set_target,
 496         .get                    = bL_cpufreq_get_rate,
 497         .init                   = bL_cpufreq_init,
 498         .have_governor_per_policy = true,
 499         .attr                   = bL_cpufreq_attr,
 500 };
 501
 502 static int bL_cpufreq_switcher_notifier(struct notifier_block *nfb,
 503                                         unsigned long action, void *_arg)
 504 {
 505         pr_debug("%s: action: %ld\n", __func__, action);
 506
 507         switch (action) {
 508         case BL_NOTIFY_PRE_ENABLE:
 509         case BL_NOTIFY_PRE_DISABLE:
 510                 cpufreq_unregister_driver(&bL_cpufreq_driver);
 511                 break;
 512
 513         case BL_NOTIFY_POST_ENABLE:
 514                 set_switching_enabled(true);
 515                 cpufreq_register_driver(&bL_cpufreq_driver);
 516                 break;
 517
 518         case BL_NOTIFY_POST_DISABLE:
 519                 set_switching_enabled(false);
 520                 cpufreq_register_driver(&bL_cpufreq_driver);
 521                 break;
 522
 523         default:
 524                 return NOTIFY_DONE;
 525         }
 526
 527         return NOTIFY_OK;
 528 }
 529
 530 static struct notifier_block bL_switcher_notifier = {
 531         .notifier_call = bL_cpufreq_switcher_notifier,
 532 };
 533
 534 int bL_cpufreq_register(struct cpufreq_arm_bL_ops *ops)
 535 {
 536         int ret, i;
 537
 538         if (arm_bL_ops) {
 539                 pr_debug("%s: Already registered: %s, exiting\n", __func__,
 540                                 arm_bL_ops->name);
 541                 return -EBUSY;
 542         }
 543
 544         if (!ops || !strlen(ops->name) || !ops->init_opp_table) {
 545                 pr_err("%s: Invalid arm_bL_ops, exiting\n", __func__);
 546                 return -ENODEV;
 547         }
 548
 549         arm_bL_ops = ops;
 550
 551         ret = bL_switcher_get_enabled();
 552         set_switching_enabled(ret);
 553
 554         for (i = 0; i < MAX_CLUSTERS; i++)
 555                 mutex_init(&cluster_lock[i]);
 556
 557         ret = cpufreq_register_driver(&bL_cpufreq_driver);
 558         if (ret) {
 559                 pr_info("%s: Failed registering platform driver: %s, err: %d\n",
 560                                 __func__, ops->name, ret);
 561                 arm_bL_ops = NULL;
 562         } else {
 563                 ret = bL_switcher_register_notifier(&bL_switcher_notifier);
 564                 if (ret) {
 565                         cpufreq_unregister_driver(&bL_cpufreq_driver);
 566                         arm_bL_ops = NULL;
 567                 } else {
 568                         pr_info("%s: Registered platform driver: %s\n",
 569                                         __func__, ops->name);
 570                 }
 571         }
 572
 573         bL_switcher_put_enabled();
 574         return ret;
 575 }
 576 EXPORT_SYMBOL_GPL(bL_cpufreq_register);
 577
 578 void bL_cpufreq_unregister(struct cpufreq_arm_bL_ops *ops)
 579 {
 580         if (arm_bL_ops != ops) {
 581                 pr_err("%s: Registered with: %s, can't unregister, exiting\n",
 582                                 __func__, arm_bL_ops->name);
 583                 return;
 584         }
 585
 586         bL_switcher_get_enabled();
 587         bL_switcher_unregister_notifier(&bL_switcher_notifier);
 588         cpufreq_unregister_driver(&bL_cpufreq_driver);
 589         bL_switcher_put_enabled();
 590         pr_info("%s: Un-registered platform driver: %s\n", __func__,
 591                         arm_bL_ops->name);
 592
 593         /* For saving table get/put on every cpu in/out */
 594         if (is_bL_switching_enabled()) {
 595                 put_cluster_clk_and_freq_table(get_cpu_device(0));
 596         } else {
 597                 int i;
 598
 599                 for (i = 0; i < MAX_CLUSTERS; i++) {
 600                         struct device *cdev = get_cpu_device(i);
 601                         if (!cdev) {
 602                                 pr_err("%s: failed to get cpu%d device\n",
 603                                                 __func__, i);
 604                                 return;
 605                         }
 606
 607                         put_cluster_clk_and_freq_table(cdev);
 608                 }
 609         }
 610
 611         arm_bL_ops = NULL;
 612 }
 613 EXPORT_SYMBOL_GPL(bL_cpufreq_unregister);