arch/arm64/kernel/topology.c

   1 /*
   2  * arch/arm64/kernel/topology.c
   3  *
   4  * Copyright (C) 2011,2013,2014 Linaro Limited.
   5  *
   6  * Based on the arm32 version written by Vincent Guittot in turn based on
   7  * arch/sh/kernel/topology.c
   8  *
   9  * This file is subject to the terms and conditions of the GNU General Public
  10  * License.  See the file "COPYING" in the main directory of this archive
  11  * for more details.
  12  */
  13
  14 #include <linux/cpu.h>
  15 #include <linux/cpumask.h>
  16 #include <linux/init.h>
  17 #include <linux/percpu.h>
  18 #include <linux/node.h>
  19 #include <linux/nodemask.h>
  20 #include <linux/of.h>
  21 #include <linux/sched.h>
  22 #include <linux/slab.h>
  23
  24 #include <asm/topology.h>
  25
  26 /*
  27  * cpu power table
  28  * This per cpu data structure describes the relative capacity of each core.
  29  * On a heteregenous system, cores don't have the same computation capacity
  30  * and we reflect that difference in the cpu_power field so the scheduler can
  31  * take this difference into account during load balance. A per cpu structure
  32  * is preferred because each CPU updates its own cpu_power field during the
  33  * load balance except for idle cores. One idle core is selected to run the
  34  * rebalance_domains for all idle cores and the cpu_power can be updated
  35  * during this sequence.
  36  */
  37 static DEFINE_PER_CPU(unsigned long, cpu_scale);
  38
  39 unsigned long arch_scale_freq_power(struct sched_domain *sd, int cpu)
  40 {
  41         return per_cpu(cpu_scale, cpu);
  42 }
  43
  44 static void set_power_scale(unsigned int cpu, unsigned long power)
  45 {
  46         per_cpu(cpu_scale, cpu) = power;
  47 }
  48
  49 static int __init get_cpu_for_node(struct device_node *node)
  50 {
  51         struct device_node *cpu_node;
  52         int cpu;
  53
  54         cpu_node = of_parse_phandle(node, "cpu", 0);
  55         if (!cpu_node)
  56                 return -1;
  57
  58         for_each_possible_cpu(cpu) {
  59                 if (of_get_cpu_node(cpu, NULL) == cpu_node) {
  60                         of_node_put(cpu_node);
  61                         return cpu;
  62                 }
  63         }
  64
  65         pr_crit("Unable to find CPU node for %s\n", cpu_node->full_name);
  66
  67         of_node_put(cpu_node);
  68         return -1;
  69 }
  70
  71 static int __init parse_core(struct device_node *core, int cluster_id,
  72                              int core_id)
  73 {
  74         char name[10];
  75         bool leaf = true;
  76         int i = 0;
  77         int cpu;
  78         struct device_node *t;
  79
  80         do {
  81                 snprintf(name, sizeof(name), "thread%d", i);
  82                 t = of_get_child_by_name(core, name);
  83                 if (t) {
  84                         leaf = false;
  85                         cpu = get_cpu_for_node(t);
  86                         if (cpu >= 0) {
  87                                 cpu_topology[cpu].cluster_id = cluster_id;
  88                                 cpu_topology[cpu].core_id = core_id;
  89                                 cpu_topology[cpu].thread_id = i;
  90                         } else {
  91                                 pr_err("%s: Can't get CPU for thread\n",
  92                                        t->full_name);
  93                                 of_node_put(t);
  94                                 return -EINVAL;
  95                         }
  96                         of_node_put(t);
  97                 }
  98                 i++;
  99         } while (t);
 100
 101         cpu = get_cpu_for_node(core);
 102         if (cpu >= 0) {
 103                 if (!leaf) {
 104                         pr_err("%s: Core has both threads and CPU\n",
 105                                core->full_name);
 106                         return -EINVAL;
 107                 }
 108
 109                 cpu_topology[cpu].cluster_id = cluster_id;
 110                 cpu_topology[cpu].core_id = core_id;
 111         } else if (leaf) {
 112                 pr_err("%s: Can't get CPU for leaf core\n", core->full_name);
 113                 return -EINVAL;
 114         }
 115
 116         return 0;
 117 }
 118
 119 static int __init parse_cluster(struct device_node *cluster, int depth)
 120 {
 121         char name[10];
 122         bool leaf = true;
 123         bool has_cores = false;
 124         struct device_node *c;
 125         static int cluster_id __initdata;
 126         int core_id = 0;
 127         int i, ret;
 128
 129         /*
 130          * First check for child clusters; we currently ignore any
 131          * information about the nesting of clusters and present the
 132          * scheduler with a flat list of them.
 133          */
 134         i = 0;
 135         do {
 136                 snprintf(name, sizeof(name), "cluster%d", i);
 137                 c = of_get_child_by_name(cluster, name);
 138                 if (c) {
 139                         leaf = false;
 140                         ret = parse_cluster(c, depth + 1);
 141                         of_node_put(c);
 142                         if (ret != 0)
 143                                 return ret;
 144                 }
 145                 i++;
 146         } while (c);
 147
 148         /* Now check for cores */
 149         i = 0;
 150         do {
 151                 snprintf(name, sizeof(name), "core%d", i);
 152                 c = of_get_child_by_name(cluster, name);
 153                 if (c) {
 154                         has_cores = true;
 155
 156                         if (depth == 0) {
 157                                 pr_err("%s: cpu-map children should be clusters\n",
 158                                        c->full_name);
 159                                 of_node_put(c);
 160                                 return -EINVAL;
 161                         }
 162
 163                         if (leaf) {
 164                                 ret = parse_core(c, cluster_id, core_id++);
 165                         } else {
 166                                 pr_err("%s: Non-leaf cluster with core %s\n",
 167                                        cluster->full_name, name);
 168                                 ret = -EINVAL;
 169                         }
 170
 171                         of_node_put(c);
 172                         if (ret != 0)
 173                                 return ret;
 174                 }
 175                 i++;
 176         } while (c);
 177
 178         if (leaf && !has_cores)
 179                 pr_warn("%s: empty cluster\n", cluster->full_name);
 180
 181         if (leaf)
 182                 cluster_id++;
 183
 184         return 0;
 185 }
 186
 187 struct cpu_efficiency {
 188         const char *compatible;
 189         unsigned long efficiency;
 190 };
 191
 192 /*
 193  * Table of relative efficiency of each processors
 194  * The efficiency value must fit in 20bit and the final
 195  * cpu_scale value must be in the range
 196  *   0 < cpu_scale < 3*SCHED_POWER_SCALE/2
 197  * in order to return at most 1 when DIV_ROUND_CLOSEST
 198  * is used to compute the capacity of a CPU.
 199  * Processors that are not defined in the table,
 200  * use the default SCHED_POWER_SCALE value for cpu_scale.
 201  */
 202 static const struct cpu_efficiency table_efficiency[] = {
 203         { "arm,cortex-a57", 3891 },
 204         { "arm,cortex-a53", 2048 },
 205         { NULL, },
 206 };
 207
 208 static unsigned long *__cpu_capacity;
 209 #define cpu_capacity(cpu)       __cpu_capacity[cpu]
 210
 211 static unsigned long middle_capacity = 1;
 212
 213 /*
 214  * Iterate all CPUs' descriptor in DT and compute the efficiency
 215  * (as per table_efficiency). Also calculate a middle efficiency
 216  * as close as possible to  (max{eff_i} - min{eff_i}) / 2
 217  * This is later used to scale the cpu_power field such that an
 218  * 'average' CPU is of middle power. Also see the comments near
 219  * table_efficiency[] and update_cpu_power().
 220  */
 221 static int __init parse_dt_topology(void)
 222 {
 223         struct device_node *cn, *map;
 224         int ret = 0;
 225         int cpu;
 226
 227         cn = of_find_node_by_path("/cpus");
 228         if (!cn) {
 229                 pr_err("No CPU information found in DT\n");
 230                 return 0;
 231         }
 232
 233         /*
 234          * When topology is provided cpu-map is essentially a root
 235          * cluster with restricted subnodes.
 236          */
 237         map = of_get_child_by_name(cn, "cpu-map");
 238         if (!map)
 239                 goto out;
 240
 241         ret = parse_cluster(map, 0);
 242         if (ret != 0)
 243                 goto out_map;
 244
 245         /*
 246          * Check that all cores are in the topology; the SMP code will
 247          * only mark cores described in the DT as possible.
 248          */
 249         for_each_possible_cpu(cpu) {
 250                 if (cpu_topology[cpu].cluster_id == -1) {
 251                         pr_err("CPU%d: No topology information specified\n",
 252                                cpu);
 253                         ret = -EINVAL;
 254                 }
 255         }
 256
 257 out_map:
 258         of_node_put(map);
 259 out:
 260         of_node_put(cn);
 261         return ret;
 262 }
 263
 264 static void __init parse_dt_cpu_power(void)
 265 {
 266         const struct cpu_efficiency *cpu_eff;
 267         struct device_node *cn;
 268         unsigned long min_capacity = ULONG_MAX;
 269         unsigned long max_capacity = 0;
 270         unsigned long capacity = 0;
 271         int cpu;
 272
 273         __cpu_capacity = kcalloc(nr_cpu_ids, sizeof(*__cpu_capacity),
 274                                  GFP_NOWAIT);
 275
 276         for_each_possible_cpu(cpu) {
 277                 const u32 *rate;
 278                 int len;
 279
 280                 /* Too early to use cpu->of_node */
 281                 cn = of_get_cpu_node(cpu, NULL);
 282                 if (!cn) {
 283                         pr_err("Missing device node for CPU %d\n", cpu);
 284                         continue;
 285                 }
 286
 287                 for (cpu_eff = table_efficiency; cpu_eff->compatible; cpu_eff++)
 288                         if (of_device_is_compatible(cn, cpu_eff->compatible))
 289                                 break;
 290
 291                 if (cpu_eff->compatible == NULL) {
 292                         pr_warn("%s: Unknown CPU type\n", cn->full_name);
 293                         continue;
 294                 }
 295
 296                 rate = of_get_property(cn, "clock-frequency", &len);
 297                 if (!rate || len != 4) {
 298                         pr_err("%s: Missing clock-frequency property\n",
 299                                 cn->full_name);
 300                         continue;
 301                 }
 302
 303                 capacity = ((be32_to_cpup(rate)) >> 20) * cpu_eff->efficiency;
 304
 305                 /* Save min capacity of the system */
 306                 if (capacity < min_capacity)
 307                         min_capacity = capacity;
 308
 309                 /* Save max capacity of the system */
 310                 if (capacity > max_capacity)
 311                         max_capacity = capacity;
 312
 313                 cpu_capacity(cpu) = capacity;
 314         }
 315
 316         /* If min and max capacities are equal we bypass the update of the
 317          * cpu_scale because all CPUs have the same capacity. Otherwise, we
 318          * compute a middle_capacity factor that will ensure that the capacity
 319          * of an 'average' CPU of the system will be as close as possible to
 320          * SCHED_POWER_SCALE, which is the default value, but with the
 321          * constraint explained near table_efficiency[].
 322          */
 323         if (min_capacity == max_capacity)
 324                 return;
 325         else if (4 * max_capacity < (3 * (max_capacity + min_capacity)))
 326                 middle_capacity = (min_capacity + max_capacity)
 327                                 >> (SCHED_POWER_SHIFT+1);
 328         else
 329                 middle_capacity = ((max_capacity / 3)
 330                                 >> (SCHED_POWER_SHIFT-1)) + 1;
 331 }
 332
 333 /*
 334  * Look for a customed capacity of a CPU in the cpu_topo_data table during the
 335  * boot. The update of all CPUs is in O(n^2) for heteregeneous system but the
 336  * function returns directly for SMP system.
 337  */
 338 static void update_cpu_power(unsigned int cpu)
 339 {
 340         if (!cpu_capacity(cpu))
 341                 return;
 342
 343         set_power_scale(cpu, cpu_capacity(cpu) / middle_capacity);
 344
 345         pr_info("CPU%u: update cpu_power %lu\n",
 346                 cpu, arch_scale_freq_power(NULL, cpu));
 347 }
 348
 349 /*
 350  * cpu topology table
 351  */
 352 struct cpu_topology cpu_topology[NR_CPUS];
 353 EXPORT_SYMBOL_GPL(cpu_topology);
 354
 355 const struct cpumask *cpu_coregroup_mask(int cpu)
 356 {
 357         return &cpu_topology[cpu].core_sibling;
 358 }
 359
 360 static void update_siblings_masks(unsigned int cpuid)
 361 {
 362         struct cpu_topology *cpu_topo, *cpuid_topo = &cpu_topology[cpuid];
 363         int cpu;
 364
 365         if (cpuid_topo->cluster_id == -1) {
 366                 /*
 367                  * DT does not contain topology information for this cpu.
 368                  */
 369                 pr_debug("CPU%u: No topology information configured\n", cpuid);
 370                 return;
 371         }
 372
 373         /* update core and thread sibling masks */
 374         for_each_possible_cpu(cpu) {
 375                 cpu_topo = &cpu_topology[cpu];
 376
 377                 if (cpuid_topo->cluster_id != cpu_topo->cluster_id)
 378                         continue;
 379
 380                 cpumask_set_cpu(cpuid, &cpu_topo->core_sibling);
 381                 if (cpu != cpuid)
 382                         cpumask_set_cpu(cpu, &cpuid_topo->core_sibling);
 383
 384                 if (cpuid_topo->core_id != cpu_topo->core_id)
 385                         continue;
 386
 387                 cpumask_set_cpu(cpuid, &cpu_topo->thread_sibling);
 388                 if (cpu != cpuid)
 389                         cpumask_set_cpu(cpu, &cpuid_topo->thread_sibling);
 390         }
 391 }
 392
 393 void store_cpu_topology(unsigned int cpuid)
 394 {
 395         update_siblings_masks(cpuid);
 396         update_cpu_power(cpuid);
 397 }
 398
 399 static void __init reset_cpu_topology(void)
 400 {
 401         unsigned int cpu;
 402
 403         for_each_possible_cpu(cpu) {
 404                 struct cpu_topology *cpu_topo = &cpu_topology[cpu];
 405
 406                 cpu_topo->thread_id = -1;
 407                 cpu_topo->core_id = 0;
 408                 cpu_topo->cluster_id = -1;
 409
 410                 cpumask_clear(&cpu_topo->core_sibling);
 411                 cpumask_set_cpu(cpu, &cpu_topo->core_sibling);
 412                 cpumask_clear(&cpu_topo->thread_sibling);
 413                 cpumask_set_cpu(cpu, &cpu_topo->thread_sibling);
 414         }
 415 }
 416
 417 static void __init reset_cpu_power(void)
 418 {
 419         unsigned int cpu;
 420
 421         for_each_possible_cpu(cpu)
 422                 set_power_scale(cpu, SCHED_POWER_SCALE);
 423 }
 424
 425 void __init init_cpu_topology(void)
 426 {
 427         reset_cpu_topology();
 428
 429         /*
 430          * Discard anything that was parsed if we hit an error so we
 431          * don't use partial information.
 432          */
 433         if (parse_dt_topology())
 434                 reset_cpu_topology();
 435
 436         reset_cpu_power();
 437         parse_dt_cpu_power();
 438 }