arch/arm64/kernel/topology.c

   1 /*
   2  * arch/arm64/kernel/topology.c
   3  *
   4  * Copyright (C) 2011,2013,2014 Linaro Limited.
   5  *
   6  * Based on the arm32 version written by Vincent Guittot in turn based on
   7  * arch/sh/kernel/topology.c
   8  *
   9  * This file is subject to the terms and conditions of the GNU General Public
  10  * License.  See the file "COPYING" in the main directory of this archive
  11  * for more details.
  12  */
  13
  14 #include <linux/acpi.h>
  15 #include <linux/arch_topology.h>
  16 #include <linux/cacheinfo.h>
  17 #include <linux/cpu.h>
  18 #include <linux/cpumask.h>
  19 #include <linux/init.h>
  20 #include <linux/percpu.h>
  21 #include <linux/node.h>
  22 #include <linux/nodemask.h>
  23 #include <linux/of.h>
  24 #include <linux/sched.h>
  25 #include <linux/sched/topology.h>
  26 #include <linux/slab.h>
  27 #include <linux/smp.h>
  28 #include <linux/string.h>
  29
  30 #include <asm/cpu.h>
  31 #include <asm/cputype.h>
  32 #include <asm/topology.h>
  33
  34 static int __init get_cpu_for_node(struct device_node *node)
  35 {
  36         struct device_node *cpu_node;
  37         int cpu;
  38
  39         cpu_node = of_parse_phandle(node, "cpu", 0);
  40         if (!cpu_node)
  41                 return -1;
  42
  43         for_each_possible_cpu(cpu) {
  44                 if (of_get_cpu_node(cpu, NULL) == cpu_node) {
  45                         topology_parse_cpu_capacity(cpu_node, cpu);
  46                         of_node_put(cpu_node);
  47                         return cpu;
  48                 }
  49         }
  50
  51         pr_crit("Unable to find CPU node for %pOF\n", cpu_node);
  52
  53         of_node_put(cpu_node);
  54         return -1;
  55 }
  56
  57 static int __init parse_core(struct device_node *core, int package_id,
  58                              int core_id)
  59 {
  60         char name[10];
  61         bool leaf = true;
  62         int i = 0;
  63         int cpu;
  64         struct device_node *t;
  65
  66         do {
  67                 snprintf(name, sizeof(name), "thread%d", i);
  68                 t = of_get_child_by_name(core, name);
  69                 if (t) {
  70                         leaf = false;
  71                         cpu = get_cpu_for_node(t);
  72                         if (cpu >= 0) {
  73                                 cpu_topology[cpu].package_id = package_id;
  74                                 cpu_topology[cpu].core_id = core_id;
  75                                 cpu_topology[cpu].thread_id = i;
  76                         } else {
  77                                 pr_err("%pOF: Can't get CPU for thread\n",
  78                                        t);
  79                                 of_node_put(t);
  80                                 return -EINVAL;
  81                         }
  82                         of_node_put(t);
  83                 }
  84                 i++;
  85         } while (t);
  86
  87         cpu = get_cpu_for_node(core);
  88         if (cpu >= 0) {
  89                 if (!leaf) {
  90                         pr_err("%pOF: Core has both threads and CPU\n",
  91                                core);
  92                         return -EINVAL;
  93                 }
  94
  95                 cpu_topology[cpu].package_id = package_id;
  96                 cpu_topology[cpu].core_id = core_id;
  97         } else if (leaf) {
  98                 pr_err("%pOF: Can't get CPU for leaf core\n", core);
  99                 return -EINVAL;
 100         }
 101
 102         return 0;
 103 }
 104
 105 static int __init parse_cluster(struct device_node *cluster, int depth)
 106 {
 107         char name[10];
 108         bool leaf = true;
 109         bool has_cores = false;
 110         struct device_node *c;
 111         static int package_id __initdata;
 112         int core_id = 0;
 113         int i, ret;
 114
 115         /*
 116          * First check for child clusters; we currently ignore any
 117          * information about the nesting of clusters and present the
 118          * scheduler with a flat list of them.
 119          */
 120         i = 0;
 121         do {
 122                 snprintf(name, sizeof(name), "cluster%d", i);
 123                 c = of_get_child_by_name(cluster, name);
 124                 if (c) {
 125                         leaf = false;
 126                         ret = parse_cluster(c, depth + 1);
 127                         of_node_put(c);
 128                         if (ret != 0)
 129                                 return ret;
 130                 }
 131                 i++;
 132         } while (c);
 133
 134         /* Now check for cores */
 135         i = 0;
 136         do {
 137                 snprintf(name, sizeof(name), "core%d", i);
 138                 c = of_get_child_by_name(cluster, name);
 139                 if (c) {
 140                         has_cores = true;
 141
 142                         if (depth == 0) {
 143                                 pr_err("%pOF: cpu-map children should be clusters\n",
 144                                        c);
 145                                 of_node_put(c);
 146                                 return -EINVAL;
 147                         }
 148
 149                         if (leaf) {
 150                                 ret = parse_core(c, package_id, core_id++);
 151                         } else {
 152                                 pr_err("%pOF: Non-leaf cluster with core %s\n",
 153                                        cluster, name);
 154                                 ret = -EINVAL;
 155                         }
 156
 157                         of_node_put(c);
 158                         if (ret != 0)
 159                                 return ret;
 160                 }
 161                 i++;
 162         } while (c);
 163
 164         if (leaf && !has_cores)
 165                 pr_warn("%pOF: empty cluster\n", cluster);
 166
 167         if (leaf)
 168                 package_id++;
 169
 170         return 0;
 171 }
 172
 173 static int __init parse_dt_topology(void)
 174 {
 175         struct device_node *cn, *map;
 176         int ret = 0;
 177         int cpu;
 178
 179         cn = of_find_node_by_path("/cpus");
 180         if (!cn) {
 181                 pr_err("No CPU information found in DT\n");
 182                 return 0;
 183         }
 184
 185         /*
 186          * When topology is provided cpu-map is essentially a root
 187          * cluster with restricted subnodes.
 188          */
 189         map = of_get_child_by_name(cn, "cpu-map");
 190         if (!map)
 191                 goto out;
 192
 193         ret = parse_cluster(map, 0);
 194         if (ret != 0)
 195                 goto out_map;
 196
 197         topology_normalize_cpu_scale();
 198
 199         /*
 200          * Check that all cores are in the topology; the SMP code will
 201          * only mark cores described in the DT as possible.
 202          */
 203         for_each_possible_cpu(cpu)
 204                 if (cpu_topology[cpu].package_id == -1)
 205                         ret = -EINVAL;
 206
 207 out_map:
 208         of_node_put(map);
 209 out:
 210         of_node_put(cn);
 211         return ret;
 212 }
 213
 214 /*
 215  * cpu topology table
 216  */
 217 struct cpu_topology cpu_topology[NR_CPUS];
 218 EXPORT_SYMBOL_GPL(cpu_topology);
 219
 220 const struct cpumask *cpu_coregroup_mask(int cpu)
 221 {
 222         const cpumask_t *core_mask = cpumask_of_node(cpu_to_node(cpu));
 223
 224         /* Find the smaller of NUMA, core or LLC siblings */
 225         if (cpumask_subset(&cpu_topology[cpu].core_sibling, core_mask)) {
 226                 /* not numa in package, lets use the package siblings */
 227                 core_mask = &cpu_topology[cpu].core_sibling;
 228         }
 229         if (cpu_topology[cpu].llc_id != -1) {
 230                 if (cpumask_subset(&cpu_topology[cpu].llc_siblings, core_mask))
 231                         core_mask = &cpu_topology[cpu].llc_siblings;
 232         }
 233
 234         return core_mask;
 235 }
 236
 237 static void update_siblings_masks(unsigned int cpuid)
 238 {
 239         struct cpu_topology *cpu_topo, *cpuid_topo = &cpu_topology[cpuid];
 240         int cpu;
 241
 242         /* update core and thread sibling masks */
 243         for_each_possible_cpu(cpu) {
 244                 cpu_topo = &cpu_topology[cpu];
 245
 246                 if (cpuid_topo->llc_id == cpu_topo->llc_id)
 247                         cpumask_set_cpu(cpu, &cpuid_topo->llc_siblings);
 248
 249                 if (cpuid_topo->package_id != cpu_topo->package_id)
 250                         continue;
 251
 252                 cpumask_set_cpu(cpuid, &cpu_topo->core_sibling);
 253                 if (cpu != cpuid)
 254                         cpumask_set_cpu(cpu, &cpuid_topo->core_sibling);
 255
 256                 if (cpuid_topo->core_id != cpu_topo->core_id)
 257                         continue;
 258
 259                 cpumask_set_cpu(cpuid, &cpu_topo->thread_sibling);
 260                 if (cpu != cpuid)
 261                         cpumask_set_cpu(cpu, &cpuid_topo->thread_sibling);
 262         }
 263 }
 264
 265 void store_cpu_topology(unsigned int cpuid)
 266 {
 267         struct cpu_topology *cpuid_topo = &cpu_topology[cpuid];
 268         u64 mpidr;
 269
 270         if (cpuid_topo->package_id != -1)
 271                 goto topology_populated;
 272
 273         mpidr = read_cpuid_mpidr();
 274
 275         /* Uniprocessor systems can rely on default topology values */
 276         if (mpidr & MPIDR_UP_BITMASK)
 277                 return;
 278
 279         /* Create cpu topology mapping based on MPIDR. */
 280         if (mpidr & MPIDR_MT_BITMASK) {
 281                 /* Multiprocessor system : Multi-threads per core */
 282                 cpuid_topo->thread_id  = MPIDR_AFFINITY_LEVEL(mpidr, 0);
 283                 cpuid_topo->core_id    = MPIDR_AFFINITY_LEVEL(mpidr, 1);
 284                 cpuid_topo->package_id = MPIDR_AFFINITY_LEVEL(mpidr, 2) |
 285                                          MPIDR_AFFINITY_LEVEL(mpidr, 3) << 8;
 286         } else {
 287                 /* Multiprocessor system : Single-thread per core */
 288                 cpuid_topo->thread_id  = -1;
 289                 cpuid_topo->core_id    = MPIDR_AFFINITY_LEVEL(mpidr, 0);
 290                 cpuid_topo->package_id = MPIDR_AFFINITY_LEVEL(mpidr, 1) |
 291                                          MPIDR_AFFINITY_LEVEL(mpidr, 2) << 8 |
 292                                          MPIDR_AFFINITY_LEVEL(mpidr, 3) << 16;
 293         }
 294
 295         pr_debug("CPU%u: cluster %d core %d thread %d mpidr %#016llx\n",
 296                  cpuid, cpuid_topo->package_id, cpuid_topo->core_id,
 297                  cpuid_topo->thread_id, mpidr);
 298
 299 topology_populated:
 300         update_siblings_masks(cpuid);
 301 }
 302
 303 static void __init reset_cpu_topology(void)
 304 {
 305         unsigned int cpu;
 306
 307         for_each_possible_cpu(cpu) {
 308                 struct cpu_topology *cpu_topo = &cpu_topology[cpu];
 309
 310                 cpu_topo->thread_id = -1;
 311                 cpu_topo->core_id = 0;
 312                 cpu_topo->package_id = -1;
 313
 314                 cpu_topo->llc_id = -1;
 315                 cpumask_clear(&cpu_topo->llc_siblings);
 316                 cpumask_set_cpu(cpu, &cpu_topo->llc_siblings);
 317
 318                 cpumask_clear(&cpu_topo->core_sibling);
 319                 cpumask_set_cpu(cpu, &cpu_topo->core_sibling);
 320                 cpumask_clear(&cpu_topo->thread_sibling);
 321                 cpumask_set_cpu(cpu, &cpu_topo->thread_sibling);
 322         }
 323 }
 324
 325 #ifdef CONFIG_ACPI
 326 /*
 327  * Propagate the topology information of the processor_topology_node tree to the
 328  * cpu_topology array.
 329  */
 330 static int __init parse_acpi_topology(void)
 331 {
 332         bool is_threaded;
 333         int cpu, topology_id;
 334
 335         is_threaded = read_cpuid_mpidr() & MPIDR_MT_BITMASK;
 336
 337         for_each_possible_cpu(cpu) {
 338                 int i, cache_id;
 339
 340                 topology_id = find_acpi_cpu_topology(cpu, 0);
 341                 if (topology_id < 0)
 342                         return topology_id;
 343
 344                 if (is_threaded) {
 345                         cpu_topology[cpu].thread_id = topology_id;
 346                         topology_id = find_acpi_cpu_topology(cpu, 1);
 347                         cpu_topology[cpu].core_id   = topology_id;
 348                 } else {
 349                         cpu_topology[cpu].thread_id  = -1;
 350                         cpu_topology[cpu].core_id    = topology_id;
 351                 }
 352                 topology_id = find_acpi_cpu_topology_package(cpu);
 353                 cpu_topology[cpu].package_id = topology_id;
 354
 355                 i = acpi_find_last_cache_level(cpu);
 356
 357                 if (i > 0) {
 358                         /*
 359                          * this is the only part of cpu_topology that has
 360                          * a direct relationship with the cache topology
 361                          */
 362                         cache_id = find_acpi_cpu_cache_topology(cpu, i);
 363                         if (cache_id > 0)
 364                                 cpu_topology[cpu].llc_id = cache_id;
 365                 }
 366         }
 367
 368         return 0;
 369 }
 370
 371 #else
 372 static inline int __init parse_acpi_topology(void)
 373 {
 374         return -EINVAL;
 375 }
 376 #endif
 377
 378 void __init init_cpu_topology(void)
 379 {
 380         reset_cpu_topology();
 381
 382         /*
 383          * Discard anything that was parsed if we hit an error so we
 384          * don't use partial information.
 385          */
 386         if (!acpi_disabled && parse_acpi_topology())
 387                 reset_cpu_topology();
 388         else if (of_have_populated_dt() && parse_dt_topology())
 389                 reset_cpu_topology();
 390 }