arch/arm64/kernel/topology.c

   1 /*
   2  * arch/arm64/kernel/topology.c
   3  *
   4  * Copyright (C) 2011,2013,2014 Linaro Limited.
   5  *
   6  * Based on the arm32 version written by Vincent Guittot in turn based on
   7  * arch/sh/kernel/topology.c
   8  *
   9  * This file is subject to the terms and conditions of the GNU General Public
  10  * License.  See the file "COPYING" in the main directory of this archive
  11  * for more details.
  12  */
  13
  14 #include <linux/arch_topology.h>
  15 #include <linux/cpu.h>
  16 #include <linux/cpumask.h>
  17 #include <linux/init.h>
  18 #include <linux/percpu.h>
  19 #include <linux/node.h>
  20 #include <linux/nodemask.h>
  21 #include <linux/of.h>
  22 #include <linux/sched.h>
  23 #include <linux/sched/topology.h>
  24 #include <linux/slab.h>
  25 #include <linux/string.h>
  26
  27 #include <asm/cpu.h>
  28 #include <asm/cputype.h>
  29 #include <asm/topology.h>
  30
  31 static int __init get_cpu_for_node(struct device_node *node)
  32 {
  33         struct device_node *cpu_node;
  34         int cpu;
  35
  36         cpu_node = of_parse_phandle(node, "cpu", 0);
  37         if (!cpu_node)
  38                 return -1;
  39
  40         for_each_possible_cpu(cpu) {
  41                 if (of_get_cpu_node(cpu, NULL) == cpu_node) {
  42                         topology_parse_cpu_capacity(cpu_node, cpu);
  43                         of_node_put(cpu_node);
  44                         return cpu;
  45                 }
  46         }
  47
  48         pr_crit("Unable to find CPU node for %s\n", cpu_node->full_name);
  49
  50         of_node_put(cpu_node);
  51         return -1;
  52 }
  53
  54 static int __init parse_core(struct device_node *core, int cluster_id,
  55                              int core_id)
  56 {
  57         char name[10];
  58         bool leaf = true;
  59         int i = 0;
  60         int cpu;
  61         struct device_node *t;
  62
  63         do {
  64                 snprintf(name, sizeof(name), "thread%d", i);
  65                 t = of_get_child_by_name(core, name);
  66                 if (t) {
  67                         leaf = false;
  68                         cpu = get_cpu_for_node(t);
  69                         if (cpu >= 0) {
  70                                 cpu_topology[cpu].cluster_id = cluster_id;
  71                                 cpu_topology[cpu].core_id = core_id;
  72                                 cpu_topology[cpu].thread_id = i;
  73                         } else {
  74                                 pr_err("%s: Can't get CPU for thread\n",
  75                                        t->full_name);
  76                                 of_node_put(t);
  77                                 return -EINVAL;
  78                         }
  79                         of_node_put(t);
  80                 }
  81                 i++;
  82         } while (t);
  83
  84         cpu = get_cpu_for_node(core);
  85         if (cpu >= 0) {
  86                 if (!leaf) {
  87                         pr_err("%s: Core has both threads and CPU\n",
  88                                core->full_name);
  89                         return -EINVAL;
  90                 }
  91
  92                 cpu_topology[cpu].cluster_id = cluster_id;
  93                 cpu_topology[cpu].core_id = core_id;
  94         } else if (leaf) {
  95                 pr_err("%s: Can't get CPU for leaf core\n", core->full_name);
  96                 return -EINVAL;
  97         }
  98
  99         return 0;
 100 }
 101
 102 static int __init parse_cluster(struct device_node *cluster, int depth)
 103 {
 104         char name[10];
 105         bool leaf = true;
 106         bool has_cores = false;
 107         struct device_node *c;
 108         static int cluster_id __initdata;
 109         int core_id = 0;
 110         int i, ret;
 111
 112         /*
 113          * First check for child clusters; we currently ignore any
 114          * information about the nesting of clusters and present the
 115          * scheduler with a flat list of them.
 116          */
 117         i = 0;
 118         do {
 119                 snprintf(name, sizeof(name), "cluster%d", i);
 120                 c = of_get_child_by_name(cluster, name);
 121                 if (c) {
 122                         leaf = false;
 123                         ret = parse_cluster(c, depth + 1);
 124                         of_node_put(c);
 125                         if (ret != 0)
 126                                 return ret;
 127                 }
 128                 i++;
 129         } while (c);
 130
 131         /* Now check for cores */
 132         i = 0;
 133         do {
 134                 snprintf(name, sizeof(name), "core%d", i);
 135                 c = of_get_child_by_name(cluster, name);
 136                 if (c) {
 137                         has_cores = true;
 138
 139                         if (depth == 0) {
 140                                 pr_err("%s: cpu-map children should be clusters\n",
 141                                        c->full_name);
 142                                 of_node_put(c);
 143                                 return -EINVAL;
 144                         }
 145
 146                         if (leaf) {
 147                                 ret = parse_core(c, cluster_id, core_id++);
 148                         } else {
 149                                 pr_err("%s: Non-leaf cluster with core %s\n",
 150                                        cluster->full_name, name);
 151                                 ret = -EINVAL;
 152                         }
 153
 154                         of_node_put(c);
 155                         if (ret != 0)
 156                                 return ret;
 157                 }
 158                 i++;
 159         } while (c);
 160
 161         if (leaf && !has_cores)
 162                 pr_warn("%s: empty cluster\n", cluster->full_name);
 163
 164         if (leaf)
 165                 cluster_id++;
 166
 167         return 0;
 168 }
 169
 170 static int __init parse_dt_topology(void)
 171 {
 172         struct device_node *cn, *map;
 173         int ret = 0;
 174         int cpu;
 175
 176         cn = of_find_node_by_path("/cpus");
 177         if (!cn) {
 178                 pr_err("No CPU information found in DT\n");
 179                 return 0;
 180         }
 181
 182         /*
 183          * When topology is provided cpu-map is essentially a root
 184          * cluster with restricted subnodes.
 185          */
 186         map = of_get_child_by_name(cn, "cpu-map");
 187         if (!map)
 188                 goto out;
 189
 190         ret = parse_cluster(map, 0);
 191         if (ret != 0)
 192                 goto out_map;
 193
 194         topology_normalize_cpu_scale();
 195
 196         /*
 197          * Check that all cores are in the topology; the SMP code will
 198          * only mark cores described in the DT as possible.
 199          */
 200         for_each_possible_cpu(cpu)
 201                 if (cpu_topology[cpu].cluster_id == -1)
 202                         ret = -EINVAL;
 203
 204 out_map:
 205         of_node_put(map);
 206 out:
 207         of_node_put(cn);
 208         return ret;
 209 }
 210
 211 /*
 212  * cpu topology table
 213  */
 214 struct cpu_topology cpu_topology[NR_CPUS];
 215 EXPORT_SYMBOL_GPL(cpu_topology);
 216
 217 const struct cpumask *cpu_coregroup_mask(int cpu)
 218 {
 219         return &cpu_topology[cpu].core_sibling;
 220 }
 221
 222 static void update_siblings_masks(unsigned int cpuid)
 223 {
 224         struct cpu_topology *cpu_topo, *cpuid_topo = &cpu_topology[cpuid];
 225         int cpu;
 226
 227         /* update core and thread sibling masks */
 228         for_each_possible_cpu(cpu) {
 229                 cpu_topo = &cpu_topology[cpu];
 230
 231                 if (cpuid_topo->cluster_id != cpu_topo->cluster_id)
 232                         continue;
 233
 234                 cpumask_set_cpu(cpuid, &cpu_topo->core_sibling);
 235                 if (cpu != cpuid)
 236                         cpumask_set_cpu(cpu, &cpuid_topo->core_sibling);
 237
 238                 if (cpuid_topo->core_id != cpu_topo->core_id)
 239                         continue;
 240
 241                 cpumask_set_cpu(cpuid, &cpu_topo->thread_sibling);
 242                 if (cpu != cpuid)
 243                         cpumask_set_cpu(cpu, &cpuid_topo->thread_sibling);
 244         }
 245 }
 246
 247 void store_cpu_topology(unsigned int cpuid)
 248 {
 249         struct cpu_topology *cpuid_topo = &cpu_topology[cpuid];
 250         u64 mpidr;
 251
 252         if (cpuid_topo->cluster_id != -1)
 253                 goto topology_populated;
 254
 255         mpidr = read_cpuid_mpidr();
 256
 257         /* Uniprocessor systems can rely on default topology values */
 258         if (mpidr & MPIDR_UP_BITMASK)
 259                 return;
 260
 261         /* Create cpu topology mapping based on MPIDR. */
 262         if (mpidr & MPIDR_MT_BITMASK) {
 263                 /* Multiprocessor system : Multi-threads per core */
 264                 cpuid_topo->thread_id  = MPIDR_AFFINITY_LEVEL(mpidr, 0);
 265                 cpuid_topo->core_id    = MPIDR_AFFINITY_LEVEL(mpidr, 1);
 266                 cpuid_topo->cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 2) |
 267                                          MPIDR_AFFINITY_LEVEL(mpidr, 3) << 8;
 268         } else {
 269                 /* Multiprocessor system : Single-thread per core */
 270                 cpuid_topo->thread_id  = -1;
 271                 cpuid_topo->core_id    = MPIDR_AFFINITY_LEVEL(mpidr, 0);
 272                 cpuid_topo->cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 1) |
 273                                          MPIDR_AFFINITY_LEVEL(mpidr, 2) << 8 |
 274                                          MPIDR_AFFINITY_LEVEL(mpidr, 3) << 16;
 275         }
 276
 277         pr_debug("CPU%u: cluster %d core %d thread %d mpidr %#016llx\n",
 278                  cpuid, cpuid_topo->cluster_id, cpuid_topo->core_id,
 279                  cpuid_topo->thread_id, mpidr);
 280
 281 topology_populated:
 282         update_siblings_masks(cpuid);
 283 }
 284
 285 static void __init reset_cpu_topology(void)
 286 {
 287         unsigned int cpu;
 288
 289         for_each_possible_cpu(cpu) {
 290                 struct cpu_topology *cpu_topo = &cpu_topology[cpu];
 291
 292                 cpu_topo->thread_id = -1;
 293                 cpu_topo->core_id = 0;
 294                 cpu_topo->cluster_id = -1;
 295
 296                 cpumask_clear(&cpu_topo->core_sibling);
 297                 cpumask_set_cpu(cpu, &cpu_topo->core_sibling);
 298                 cpumask_clear(&cpu_topo->thread_sibling);
 299                 cpumask_set_cpu(cpu, &cpu_topo->thread_sibling);
 300         }
 301 }
 302
 303 void __init init_cpu_topology(void)
 304 {
 305         reset_cpu_topology();
 306
 307         /*
 308          * Discard anything that was parsed if we hit an error so we
 309          * don't use partial information.
 310          */
 311         if (of_have_populated_dt() && parse_dt_topology())
 312                 reset_cpu_topology();
 313 }