2 * arch/arm/kernel/topology.c
4 * Copyright (C) 2011 Linaro Limited.
5 * Written by: Vincent Guittot
7 * based on arch/sh/kernel/topology.c
9 * This file is subject to the terms and conditions of the GNU General Public
10 * License. See the file "COPYING" in the main directory of this archive
14 #include <linux/cpu.h>
15 #include <linux/cpufreq.h>
16 #include <linux/cpumask.h>
17 #include <linux/export.h>
18 #include <linux/init.h>
19 #include <linux/percpu.h>
20 #include <linux/node.h>
21 #include <linux/nodemask.h>
23 #include <linux/sched.h>
24 #include <linux/sched/topology.h>
25 #include <linux/slab.h>
26 #include <linux/string.h>
29 #include <asm/cputype.h>
30 #include <asm/topology.h>
33 * cpu capacity scale management
38 * This per cpu data structure describes the relative capacity of each core.
39 * On a heteregenous system, cores don't have the same computation capacity
40 * and we reflect that difference in the cpu_capacity field so the scheduler
41 * can take this difference into account during load balance. A per cpu
42 * structure is preferred because each CPU updates its own cpu_capacity field
43 * during the load balance except for idle cores. One idle core is selected
44 * to run the rebalance_domains for all idle cores and the cpu_capacity can be
45 * updated during this sequence.
47 static DEFINE_PER_CPU(unsigned long, cpu_scale
) = SCHED_CAPACITY_SCALE
;
48 static DEFINE_MUTEX(cpu_scale_mutex
);
50 unsigned long arch_scale_cpu_capacity(struct sched_domain
*sd
, int cpu
)
52 return per_cpu(cpu_scale
, cpu
);
55 static void set_capacity_scale(unsigned int cpu
, unsigned long capacity
)
57 per_cpu(cpu_scale
, cpu
) = capacity
;
60 #ifdef CONFIG_PROC_SYSCTL
61 static ssize_t
cpu_capacity_show(struct device
*dev
,
62 struct device_attribute
*attr
,
65 struct cpu
*cpu
= container_of(dev
, struct cpu
, dev
);
67 return sprintf(buf
, "%lu\n",
68 arch_scale_cpu_capacity(NULL
, cpu
->dev
.id
));
71 static ssize_t
cpu_capacity_store(struct device
*dev
,
72 struct device_attribute
*attr
,
76 struct cpu
*cpu
= container_of(dev
, struct cpu
, dev
);
77 int this_cpu
= cpu
->dev
.id
, i
;
78 unsigned long new_capacity
;
82 ret
= kstrtoul(buf
, 0, &new_capacity
);
85 if (new_capacity
> SCHED_CAPACITY_SCALE
)
88 mutex_lock(&cpu_scale_mutex
);
89 for_each_cpu(i
, &cpu_topology
[this_cpu
].core_sibling
)
90 set_capacity_scale(i
, new_capacity
);
91 mutex_unlock(&cpu_scale_mutex
);
97 static DEVICE_ATTR_RW(cpu_capacity
);
99 static int register_cpu_capacity_sysctl(void)
104 for_each_possible_cpu(i
) {
105 cpu
= get_cpu_device(i
);
107 pr_err("%s: too early to get CPU%d device!\n",
111 device_create_file(cpu
, &dev_attr_cpu_capacity
);
116 subsys_initcall(register_cpu_capacity_sysctl
);
120 struct cpu_efficiency
{
121 const char *compatible
;
122 unsigned long efficiency
;
126 * Table of relative efficiency of each processors
127 * The efficiency value must fit in 20bit and the final
128 * cpu_scale value must be in the range
129 * 0 < cpu_scale < 3*SCHED_CAPACITY_SCALE/2
130 * in order to return at most 1 when DIV_ROUND_CLOSEST
131 * is used to compute the capacity of a CPU.
132 * Processors that are not defined in the table,
133 * use the default SCHED_CAPACITY_SCALE value for cpu_scale.
135 static const struct cpu_efficiency table_efficiency
[] = {
136 {"arm,cortex-a15", 3891},
137 {"arm,cortex-a7", 2048},
141 static unsigned long *__cpu_capacity
;
142 #define cpu_capacity(cpu) __cpu_capacity[cpu]
144 static unsigned long middle_capacity
= 1;
145 static bool cap_from_dt
= true;
146 static u32
*raw_capacity
;
147 static bool cap_parsing_failed
;
148 static u32 capacity_scale
;
150 static int __init
parse_cpu_capacity(struct device_node
*cpu_node
, int cpu
)
155 if (cap_parsing_failed
)
158 ret
= of_property_read_u32(cpu_node
,
159 "capacity-dmips-mhz",
163 raw_capacity
= kcalloc(num_possible_cpus(),
164 sizeof(*raw_capacity
),
167 pr_err("cpu_capacity: failed to allocate memory for raw capacities\n");
168 cap_parsing_failed
= true;
172 capacity_scale
= max(cpu_capacity
, capacity_scale
);
173 raw_capacity
[cpu
] = cpu_capacity
;
174 pr_debug("cpu_capacity: %s cpu_capacity=%u (raw)\n",
175 cpu_node
->full_name
, raw_capacity
[cpu
]);
178 pr_err("cpu_capacity: missing %s raw capacity\n",
179 cpu_node
->full_name
);
180 pr_err("cpu_capacity: partial information: fallback to 1024 for all CPUs\n");
182 cap_parsing_failed
= true;
189 static void normalize_cpu_capacity(void)
194 if (!raw_capacity
|| cap_parsing_failed
)
197 pr_debug("cpu_capacity: capacity_scale=%u\n", capacity_scale
);
198 mutex_lock(&cpu_scale_mutex
);
199 for_each_possible_cpu(cpu
) {
200 capacity
= (raw_capacity
[cpu
] << SCHED_CAPACITY_SHIFT
)
202 set_capacity_scale(cpu
, capacity
);
203 pr_debug("cpu_capacity: CPU%d cpu_capacity=%lu\n",
204 cpu
, arch_scale_cpu_capacity(NULL
, cpu
));
206 mutex_unlock(&cpu_scale_mutex
);
209 #ifdef CONFIG_CPU_FREQ
210 static cpumask_var_t cpus_to_visit
;
211 static bool cap_parsing_done
;
212 static void parsing_done_workfn(struct work_struct
*work
);
213 static DECLARE_WORK(parsing_done_work
, parsing_done_workfn
);
216 init_cpu_capacity_callback(struct notifier_block
*nb
,
220 struct cpufreq_policy
*policy
= data
;
223 if (cap_parsing_failed
|| cap_parsing_done
)
228 pr_debug("cpu_capacity: init cpu capacity for CPUs [%*pbl] (to_visit=%*pbl)\n",
229 cpumask_pr_args(policy
->related_cpus
),
230 cpumask_pr_args(cpus_to_visit
));
231 cpumask_andnot(cpus_to_visit
,
233 policy
->related_cpus
);
234 for_each_cpu(cpu
, policy
->related_cpus
) {
235 raw_capacity
[cpu
] = arch_scale_cpu_capacity(NULL
, cpu
) *
236 policy
->cpuinfo
.max_freq
/ 1000UL;
237 capacity_scale
= max(raw_capacity
[cpu
], capacity_scale
);
239 if (cpumask_empty(cpus_to_visit
)) {
240 normalize_cpu_capacity();
242 pr_debug("cpu_capacity: parsing done\n");
243 cap_parsing_done
= true;
244 schedule_work(&parsing_done_work
);
250 static struct notifier_block init_cpu_capacity_notifier
= {
251 .notifier_call
= init_cpu_capacity_callback
,
254 static int __init
register_cpufreq_notifier(void)
256 if (cap_parsing_failed
)
259 if (!alloc_cpumask_var(&cpus_to_visit
, GFP_KERNEL
)) {
260 pr_err("cpu_capacity: failed to allocate memory for cpus_to_visit\n");
263 cpumask_copy(cpus_to_visit
, cpu_possible_mask
);
265 return cpufreq_register_notifier(&init_cpu_capacity_notifier
,
266 CPUFREQ_POLICY_NOTIFIER
);
268 core_initcall(register_cpufreq_notifier
);
270 static void parsing_done_workfn(struct work_struct
*work
)
272 cpufreq_unregister_notifier(&init_cpu_capacity_notifier
,
273 CPUFREQ_POLICY_NOTIFIER
);
277 static int __init
free_raw_capacity(void)
283 core_initcall(free_raw_capacity
);
287 * Iterate all CPUs' descriptor in DT and compute the efficiency
288 * (as per table_efficiency). Also calculate a middle efficiency
289 * as close as possible to (max{eff_i} - min{eff_i}) / 2
290 * This is later used to scale the cpu_capacity field such that an
291 * 'average' CPU is of middle capacity. Also see the comments near
292 * table_efficiency[] and update_cpu_capacity().
294 static void __init
parse_dt_topology(void)
296 const struct cpu_efficiency
*cpu_eff
;
297 struct device_node
*cn
= NULL
;
298 unsigned long min_capacity
= ULONG_MAX
;
299 unsigned long max_capacity
= 0;
300 unsigned long capacity
= 0;
303 __cpu_capacity
= kcalloc(nr_cpu_ids
, sizeof(*__cpu_capacity
),
306 cn
= of_find_node_by_path("/cpus");
308 pr_err("No CPU information found in DT\n");
312 for_each_possible_cpu(cpu
) {
316 /* too early to use cpu->of_node */
317 cn
= of_get_cpu_node(cpu
, NULL
);
319 pr_err("missing device node for CPU %d\n", cpu
);
323 if (parse_cpu_capacity(cn
, cpu
)) {
330 for (cpu_eff
= table_efficiency
; cpu_eff
->compatible
; cpu_eff
++)
331 if (of_device_is_compatible(cn
, cpu_eff
->compatible
))
334 if (cpu_eff
->compatible
== NULL
)
337 rate
= of_get_property(cn
, "clock-frequency", &len
);
338 if (!rate
|| len
!= 4) {
339 pr_err("%s missing clock-frequency property\n",
344 capacity
= ((be32_to_cpup(rate
)) >> 20) * cpu_eff
->efficiency
;
346 /* Save min capacity of the system */
347 if (capacity
< min_capacity
)
348 min_capacity
= capacity
;
350 /* Save max capacity of the system */
351 if (capacity
> max_capacity
)
352 max_capacity
= capacity
;
354 cpu_capacity(cpu
) = capacity
;
357 /* If min and max capacities are equals, we bypass the update of the
358 * cpu_scale because all CPUs have the same capacity. Otherwise, we
359 * compute a middle_capacity factor that will ensure that the capacity
360 * of an 'average' CPU of the system will be as close as possible to
361 * SCHED_CAPACITY_SCALE, which is the default value, but with the
362 * constraint explained near table_efficiency[].
364 if (4*max_capacity
< (3*(max_capacity
+ min_capacity
)))
365 middle_capacity
= (min_capacity
+ max_capacity
)
366 >> (SCHED_CAPACITY_SHIFT
+1);
368 middle_capacity
= ((max_capacity
/ 3)
369 >> (SCHED_CAPACITY_SHIFT
-1)) + 1;
371 if (cap_from_dt
&& !cap_parsing_failed
)
372 normalize_cpu_capacity();
376 * Look for a customed capacity of a CPU in the cpu_capacity table during the
377 * boot. The update of all CPUs is in O(n^2) for heteregeneous system but the
378 * function returns directly for SMP system.
380 static void update_cpu_capacity(unsigned int cpu
)
382 if (!cpu_capacity(cpu
) || cap_from_dt
)
385 set_capacity_scale(cpu
, cpu_capacity(cpu
) / middle_capacity
);
387 pr_info("CPU%u: update cpu_capacity %lu\n",
388 cpu
, arch_scale_cpu_capacity(NULL
, cpu
));
392 static inline void parse_dt_topology(void) {}
393 static inline void update_cpu_capacity(unsigned int cpuid
) {}
399 struct cputopo_arm cpu_topology
[NR_CPUS
];
400 EXPORT_SYMBOL_GPL(cpu_topology
);
402 const struct cpumask
*cpu_coregroup_mask(int cpu
)
404 return &cpu_topology
[cpu
].core_sibling
;
408 * The current assumption is that we can power gate each core independently.
409 * This will be superseded by DT binding once available.
411 const struct cpumask
*cpu_corepower_mask(int cpu
)
413 return &cpu_topology
[cpu
].thread_sibling
;
416 static void update_siblings_masks(unsigned int cpuid
)
418 struct cputopo_arm
*cpu_topo
, *cpuid_topo
= &cpu_topology
[cpuid
];
421 /* update core and thread sibling masks */
422 for_each_possible_cpu(cpu
) {
423 cpu_topo
= &cpu_topology
[cpu
];
425 if (cpuid_topo
->socket_id
!= cpu_topo
->socket_id
)
428 cpumask_set_cpu(cpuid
, &cpu_topo
->core_sibling
);
430 cpumask_set_cpu(cpu
, &cpuid_topo
->core_sibling
);
432 if (cpuid_topo
->core_id
!= cpu_topo
->core_id
)
435 cpumask_set_cpu(cpuid
, &cpu_topo
->thread_sibling
);
437 cpumask_set_cpu(cpu
, &cpuid_topo
->thread_sibling
);
443 * store_cpu_topology is called at boot when only one cpu is running
444 * and with the mutex cpu_hotplug.lock locked, when several cpus have booted,
445 * which prevents simultaneous write access to cpu_topology array
447 void store_cpu_topology(unsigned int cpuid
)
449 struct cputopo_arm
*cpuid_topo
= &cpu_topology
[cpuid
];
452 /* If the cpu topology has been already set, just return */
453 if (cpuid_topo
->core_id
!= -1)
456 mpidr
= read_cpuid_mpidr();
458 /* create cpu topology mapping */
459 if ((mpidr
& MPIDR_SMP_BITMASK
) == MPIDR_SMP_VALUE
) {
461 * This is a multiprocessor system
462 * multiprocessor format & multiprocessor mode field are set
465 if (mpidr
& MPIDR_MT_BITMASK
) {
466 /* core performance interdependency */
467 cpuid_topo
->thread_id
= MPIDR_AFFINITY_LEVEL(mpidr
, 0);
468 cpuid_topo
->core_id
= MPIDR_AFFINITY_LEVEL(mpidr
, 1);
469 cpuid_topo
->socket_id
= MPIDR_AFFINITY_LEVEL(mpidr
, 2);
471 /* largely independent cores */
472 cpuid_topo
->thread_id
= -1;
473 cpuid_topo
->core_id
= MPIDR_AFFINITY_LEVEL(mpidr
, 0);
474 cpuid_topo
->socket_id
= MPIDR_AFFINITY_LEVEL(mpidr
, 1);
478 * This is an uniprocessor system
479 * we are in multiprocessor format but uniprocessor system
480 * or in the old uniprocessor format
482 cpuid_topo
->thread_id
= -1;
483 cpuid_topo
->core_id
= 0;
484 cpuid_topo
->socket_id
= -1;
487 update_siblings_masks(cpuid
);
489 update_cpu_capacity(cpuid
);
491 pr_info("CPU%u: thread %d, cpu %d, socket %d, mpidr %x\n",
492 cpuid
, cpu_topology
[cpuid
].thread_id
,
493 cpu_topology
[cpuid
].core_id
,
494 cpu_topology
[cpuid
].socket_id
, mpidr
);
497 static inline int cpu_corepower_flags(void)
499 return SD_SHARE_PKG_RESOURCES
| SD_SHARE_POWERDOMAIN
;
502 static struct sched_domain_topology_level arm_topology
[] = {
503 #ifdef CONFIG_SCHED_MC
504 { cpu_corepower_mask
, cpu_corepower_flags
, SD_INIT_NAME(GMC
) },
505 { cpu_coregroup_mask
, cpu_core_flags
, SD_INIT_NAME(MC
) },
507 { cpu_cpu_mask
, SD_INIT_NAME(DIE
) },
512 * init_cpu_topology is called at boot when only one cpu is running
513 * which prevent simultaneous write access to cpu_topology array
515 void __init
init_cpu_topology(void)
519 /* init core mask and capacity */
520 for_each_possible_cpu(cpu
) {
521 struct cputopo_arm
*cpu_topo
= &(cpu_topology
[cpu
]);
523 cpu_topo
->thread_id
= -1;
524 cpu_topo
->core_id
= -1;
525 cpu_topo
->socket_id
= -1;
526 cpumask_clear(&cpu_topo
->core_sibling
);
527 cpumask_clear(&cpu_topo
->thread_sibling
);
533 /* Set scheduler topology descriptor */
534 set_sched_topology(arm_topology
);