arch/arm64/kernel/topology.c

   1 /*
   2  * arch/arm64/kernel/topology.c
   3  *
   4  * Copyright (C) 2011,2013,2014 Linaro Limited.
   5  *
   6  * Based on the arm32 version written by Vincent Guittot in turn based on
   7  * arch/sh/kernel/topology.c
   8  *
   9  * This file is subject to the terms and conditions of the GNU General Public
  10  * License.  See the file "COPYING" in the main directory of this archive
  11  * for more details.
  12  */
  13
  14 #include <linux/acpi.h>
  15 #include <linux/arch_topology.h>
  16 #include <linux/cacheinfo.h>
  17 #include <linux/cpu.h>
  18 #include <linux/cpumask.h>
  19 #include <linux/init.h>
  20 #include <linux/percpu.h>
  21 #include <linux/node.h>
  22 #include <linux/nodemask.h>
  23 #include <linux/of.h>
  24 #include <linux/sched.h>
  25 #include <linux/sched/topology.h>
  26 #include <linux/slab.h>
  27 #include <linux/smp.h>
  28 #include <linux/string.h>
  29
  30 #include <asm/cpu.h>
  31 #include <asm/cputype.h>
  32 #include <asm/topology.h>
  33
  34 static int __init get_cpu_for_node(struct device_node *node)
  35 {
  36         struct device_node *cpu_node;
  37         int cpu;
  38
  39         cpu_node = of_parse_phandle(node, "cpu", 0);
  40         if (!cpu_node)
  41                 return -1;
  42
  43         for_each_possible_cpu(cpu) {
  44                 if (of_get_cpu_node(cpu, NULL) == cpu_node) {
  45                         topology_parse_cpu_capacity(cpu_node, cpu);
  46                         of_node_put(cpu_node);
  47                         return cpu;
  48                 }
  49         }
  50
  51         pr_crit("Unable to find CPU node for %pOF\n", cpu_node);
  52
  53         of_node_put(cpu_node);
  54         return -1;
  55 }
  56
  57 static int __init parse_core(struct device_node *core, int package_id,
  58                              int core_id)
  59 {
  60         char name[10];
  61         bool leaf = true;
  62         int i = 0;
  63         int cpu;
  64         struct device_node *t;
  65
  66         do {
  67                 snprintf(name, sizeof(name), "thread%d", i);
  68                 t = of_get_child_by_name(core, name);
  69                 if (t) {
  70                         leaf = false;
  71                         cpu = get_cpu_for_node(t);
  72                         if (cpu >= 0) {
  73                                 cpu_topology[cpu].package_id = package_id;
  74                                 cpu_topology[cpu].core_id = core_id;
  75                                 cpu_topology[cpu].thread_id = i;
  76                         } else {
  77                                 pr_err("%pOF: Can't get CPU for thread\n",
  78                                        t);
  79                                 of_node_put(t);
  80                                 return -EINVAL;
  81                         }
  82                         of_node_put(t);
  83                 }
  84                 i++;
  85         } while (t);
  86
  87         cpu = get_cpu_for_node(core);
  88         if (cpu >= 0) {
  89                 if (!leaf) {
  90                         pr_err("%pOF: Core has both threads and CPU\n",
  91                                core);
  92                         return -EINVAL;
  93                 }
  94
  95                 cpu_topology[cpu].package_id = package_id;
  96                 cpu_topology[cpu].core_id = core_id;
  97         } else if (leaf) {
  98                 pr_err("%pOF: Can't get CPU for leaf core\n", core);
  99                 return -EINVAL;
 100         }
 101
 102         return 0;
 103 }
 104
 105 static int __init parse_cluster(struct device_node *cluster, int depth)
 106 {
 107         char name[10];
 108         bool leaf = true;
 109         bool has_cores = false;
 110         struct device_node *c;
 111         static int package_id __initdata;
 112         int core_id = 0;
 113         int i, ret;
 114
 115         /*
 116          * First check for child clusters; we currently ignore any
 117          * information about the nesting of clusters and present the
 118          * scheduler with a flat list of them.
 119          */
 120         i = 0;
 121         do {
 122                 snprintf(name, sizeof(name), "cluster%d", i);
 123                 c = of_get_child_by_name(cluster, name);
 124                 if (c) {
 125                         leaf = false;
 126                         ret = parse_cluster(c, depth + 1);
 127                         of_node_put(c);
 128                         if (ret != 0)
 129                                 return ret;
 130                 }
 131                 i++;
 132         } while (c);
 133
 134         /* Now check for cores */
 135         i = 0;
 136         do {
 137                 snprintf(name, sizeof(name), "core%d", i);
 138                 c = of_get_child_by_name(cluster, name);
 139                 if (c) {
 140                         has_cores = true;
 141
 142                         if (depth == 0) {
 143                                 pr_err("%pOF: cpu-map children should be clusters\n",
 144                                        c);
 145                                 of_node_put(c);
 146                                 return -EINVAL;
 147                         }
 148
 149                         if (leaf) {
 150                                 ret = parse_core(c, package_id, core_id++);
 151                         } else {
 152                                 pr_err("%pOF: Non-leaf cluster with core %s\n",
 153                                        cluster, name);
 154                                 ret = -EINVAL;
 155                         }
 156
 157                         of_node_put(c);
 158                         if (ret != 0)
 159                                 return ret;
 160                 }
 161                 i++;
 162         } while (c);
 163
 164         if (leaf && !has_cores)
 165                 pr_warn("%pOF: empty cluster\n", cluster);
 166
 167         if (leaf)
 168                 package_id++;
 169
 170         return 0;
 171 }
 172
 173 static int __init parse_dt_topology(void)
 174 {
 175         struct device_node *cn, *map;
 176         int ret = 0;
 177         int cpu;
 178
 179         cn = of_find_node_by_path("/cpus");
 180         if (!cn) {
 181                 pr_err("No CPU information found in DT\n");
 182                 return 0;
 183         }
 184
 185         /*
 186          * When topology is provided cpu-map is essentially a root
 187          * cluster with restricted subnodes.
 188          */
 189         map = of_get_child_by_name(cn, "cpu-map");
 190         if (!map)
 191                 goto out;
 192
 193         ret = parse_cluster(map, 0);
 194         if (ret != 0)
 195                 goto out_map;
 196
 197         topology_normalize_cpu_scale();
 198
 199         /*
 200          * Check that all cores are in the topology; the SMP code will
 201          * only mark cores described in the DT as possible.
 202          */
 203         for_each_possible_cpu(cpu)
 204                 if (cpu_topology[cpu].package_id == -1)
 205                         ret = -EINVAL;
 206
 207 out_map:
 208         of_node_put(map);
 209 out:
 210         of_node_put(cn);
 211         return ret;
 212 }
 213
 214 /*
 215  * cpu topology table
 216  */
 217 struct cpu_topology cpu_topology[NR_CPUS];
 218 EXPORT_SYMBOL_GPL(cpu_topology);
 219
 220 const struct cpumask *cpu_coregroup_mask(int cpu)
 221 {
 222         const cpumask_t *core_mask = &cpu_topology[cpu].core_sibling;
 223
 224         if (cpu_topology[cpu].llc_id != -1) {
 225                 if (cpumask_subset(&cpu_topology[cpu].llc_siblings, core_mask))
 226                         core_mask = &cpu_topology[cpu].llc_siblings;
 227         }
 228
 229         return core_mask;
 230 }
 231
 232 static void update_siblings_masks(unsigned int cpuid)
 233 {
 234         struct cpu_topology *cpu_topo, *cpuid_topo = &cpu_topology[cpuid];
 235         int cpu;
 236
 237         /* update core and thread sibling masks */
 238         for_each_possible_cpu(cpu) {
 239                 cpu_topo = &cpu_topology[cpu];
 240
 241                 if (cpuid_topo->llc_id == cpu_topo->llc_id) {
 242                         cpumask_set_cpu(cpu, &cpuid_topo->llc_siblings);
 243                         cpumask_set_cpu(cpuid, &cpu_topo->llc_siblings);
 244                 }
 245
 246                 if (cpuid_topo->package_id != cpu_topo->package_id)
 247                         continue;
 248
 249                 cpumask_set_cpu(cpuid, &cpu_topo->core_sibling);
 250                 if (cpu != cpuid)
 251                         cpumask_set_cpu(cpu, &cpuid_topo->core_sibling);
 252
 253                 if (cpuid_topo->core_id != cpu_topo->core_id)
 254                         continue;
 255
 256                 cpumask_set_cpu(cpuid, &cpu_topo->thread_sibling);
 257                 if (cpu != cpuid)
 258                         cpumask_set_cpu(cpu, &cpuid_topo->thread_sibling);
 259         }
 260 }
 261
 262 void store_cpu_topology(unsigned int cpuid)
 263 {
 264         struct cpu_topology *cpuid_topo = &cpu_topology[cpuid];
 265         u64 mpidr;
 266
 267         if (cpuid_topo->package_id != -1)
 268                 goto topology_populated;
 269
 270         mpidr = read_cpuid_mpidr();
 271
 272         /* Uniprocessor systems can rely on default topology values */
 273         if (mpidr & MPIDR_UP_BITMASK)
 274                 return;
 275
 276         /* Create cpu topology mapping based on MPIDR. */
 277         if (mpidr & MPIDR_MT_BITMASK) {
 278                 /* Multiprocessor system : Multi-threads per core */
 279                 cpuid_topo->thread_id  = MPIDR_AFFINITY_LEVEL(mpidr, 0);
 280                 cpuid_topo->core_id    = MPIDR_AFFINITY_LEVEL(mpidr, 1);
 281                 cpuid_topo->package_id = MPIDR_AFFINITY_LEVEL(mpidr, 2) |
 282                                          MPIDR_AFFINITY_LEVEL(mpidr, 3) << 8;
 283         } else {
 284                 /* Multiprocessor system : Single-thread per core */
 285                 cpuid_topo->thread_id  = -1;
 286                 cpuid_topo->core_id    = MPIDR_AFFINITY_LEVEL(mpidr, 0);
 287                 cpuid_topo->package_id = MPIDR_AFFINITY_LEVEL(mpidr, 1) |
 288                                          MPIDR_AFFINITY_LEVEL(mpidr, 2) << 8 |
 289                                          MPIDR_AFFINITY_LEVEL(mpidr, 3) << 16;
 290         }
 291
 292         pr_debug("CPU%u: cluster %d core %d thread %d mpidr %#016llx\n",
 293                  cpuid, cpuid_topo->package_id, cpuid_topo->core_id,
 294                  cpuid_topo->thread_id, mpidr);
 295
 296 topology_populated:
 297         update_siblings_masks(cpuid);
 298 }
 299
 300 static void __init reset_cpu_topology(void)
 301 {
 302         unsigned int cpu;
 303
 304         for_each_possible_cpu(cpu) {
 305                 struct cpu_topology *cpu_topo = &cpu_topology[cpu];
 306
 307                 cpu_topo->thread_id = -1;
 308                 cpu_topo->core_id = 0;
 309                 cpu_topo->package_id = -1;
 310
 311                 cpu_topo->llc_id = -1;
 312                 cpumask_clear(&cpu_topo->llc_siblings);
 313                 cpumask_set_cpu(cpu, &cpu_topo->llc_siblings);
 314
 315                 cpumask_clear(&cpu_topo->core_sibling);
 316                 cpumask_set_cpu(cpu, &cpu_topo->core_sibling);
 317                 cpumask_clear(&cpu_topo->thread_sibling);
 318                 cpumask_set_cpu(cpu, &cpu_topo->thread_sibling);
 319         }
 320 }
 321
 322 #ifdef CONFIG_ACPI
 323 /*
 324  * Propagate the topology information of the processor_topology_node tree to the
 325  * cpu_topology array.
 326  */
 327 static int __init parse_acpi_topology(void)
 328 {
 329         bool is_threaded;
 330         int cpu, topology_id;
 331
 332         is_threaded = read_cpuid_mpidr() & MPIDR_MT_BITMASK;
 333
 334         for_each_possible_cpu(cpu) {
 335                 int i, cache_id;
 336
 337                 topology_id = find_acpi_cpu_topology(cpu, 0);
 338                 if (topology_id < 0)
 339                         return topology_id;
 340
 341                 if (is_threaded) {
 342                         cpu_topology[cpu].thread_id = topology_id;
 343                         topology_id = find_acpi_cpu_topology(cpu, 1);
 344                         cpu_topology[cpu].core_id   = topology_id;
 345                 } else {
 346                         cpu_topology[cpu].thread_id  = -1;
 347                         cpu_topology[cpu].core_id    = topology_id;
 348                 }
 349                 topology_id = find_acpi_cpu_topology_package(cpu);
 350                 cpu_topology[cpu].package_id = topology_id;
 351
 352                 i = acpi_find_last_cache_level(cpu);
 353
 354                 if (i > 0) {
 355                         /*
 356                          * this is the only part of cpu_topology that has
 357                          * a direct relationship with the cache topology
 358                          */
 359                         cache_id = find_acpi_cpu_cache_topology(cpu, i);
 360                         if (cache_id > 0)
 361                                 cpu_topology[cpu].llc_id = cache_id;
 362                 }
 363         }
 364
 365         return 0;
 366 }
 367
 368 #else
 369 static inline int __init parse_acpi_topology(void)
 370 {
 371         return -EINVAL;
 372 }
 373 #endif
 374
 375 void __init init_cpu_topology(void)
 376 {
 377         reset_cpu_topology();
 378
 379         /*
 380          * Discard anything that was parsed if we hit an error so we
 381          * don't use partial information.
 382          */
 383         if (!acpi_disabled && parse_acpi_topology())
 384                 reset_cpu_topology();
 385         else if (of_have_populated_dt() && parse_dt_topology())
 386                 reset_cpu_topology();
 387 }