2 * Copyright IBM Corp. 2007, 2011
3 * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>
6 #define KMSG_COMPONENT "cpu"
7 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
9 #include <linux/workqueue.h>
10 #include <linux/bootmem.h>
11 #include <linux/uaccess.h>
12 #include <linux/sysctl.h>
13 #include <linux/cpuset.h>
14 #include <linux/device.h>
15 #include <linux/export.h>
16 #include <linux/kernel.h>
17 #include <linux/sched.h>
18 #include <linux/sched/topology.h>
19 #include <linux/delay.h>
20 #include <linux/init.h>
21 #include <linux/slab.h>
22 #include <linux/cpu.h>
23 #include <linux/smp.h>
25 #include <linux/nodemask.h>
26 #include <linux/node.h>
27 #include <asm/sysinfo.h>
30 #define PTF_HORIZONTAL (0UL)
31 #define PTF_VERTICAL (1UL)
32 #define PTF_CHECK (2UL)
37 TOPOLOGY_MODE_PACKAGE
,
38 TOPOLOGY_MODE_UNINITIALIZED
42 struct mask_info
*next
;
47 static int topology_mode
= TOPOLOGY_MODE_UNINITIALIZED
;
48 static void set_topology_timer(void);
49 static void topology_work_fn(struct work_struct
*work
);
50 static struct sysinfo_15_1_x
*tl_info
;
52 static DECLARE_WORK(topology_work
, topology_work_fn
);
55 * Socket/Book linked lists and cpu_topology updates are
56 * protected by "sched_domains_mutex".
58 static struct mask_info socket_info
;
59 static struct mask_info book_info
;
60 static struct mask_info drawer_info
;
62 struct cpu_topology_s390 cpu_topology
[NR_CPUS
];
63 EXPORT_SYMBOL_GPL(cpu_topology
);
65 cpumask_t cpus_with_topology
;
67 static cpumask_t
cpu_group_map(struct mask_info
*info
, unsigned int cpu
)
71 cpumask_copy(&mask
, cpumask_of(cpu
));
72 switch (topology_mode
) {
73 case TOPOLOGY_MODE_HW
:
75 if (cpumask_test_cpu(cpu
, &info
->mask
)) {
81 if (cpumask_empty(&mask
))
82 cpumask_copy(&mask
, cpumask_of(cpu
));
84 case TOPOLOGY_MODE_PACKAGE
:
85 cpumask_copy(&mask
, cpu_present_mask
);
89 case TOPOLOGY_MODE_SINGLE
:
90 cpumask_copy(&mask
, cpumask_of(cpu
));
96 static cpumask_t
cpu_thread_map(unsigned int cpu
)
101 cpumask_copy(&mask
, cpumask_of(cpu
));
102 if (topology_mode
!= TOPOLOGY_MODE_HW
)
104 cpu
-= cpu
% (smp_cpu_mtid
+ 1);
105 for (i
= 0; i
<= smp_cpu_mtid
; i
++)
106 if (cpu_present(cpu
+ i
))
107 cpumask_set_cpu(cpu
+ i
, &mask
);
111 #define TOPOLOGY_CORE_BITS 64
113 static void add_cpus_to_mask(struct topology_core
*tl_core
,
114 struct mask_info
*drawer
,
115 struct mask_info
*book
,
116 struct mask_info
*socket
)
118 struct cpu_topology_s390
*topo
;
121 for_each_set_bit(core
, &tl_core
->mask
, TOPOLOGY_CORE_BITS
) {
125 rcore
= TOPOLOGY_CORE_BITS
- 1 - core
+ tl_core
->origin
;
126 lcpu
= smp_find_processor_id(rcore
<< smp_cpu_mt_shift
);
129 for (i
= 0; i
<= smp_cpu_mtid
; i
++) {
130 topo
= &cpu_topology
[lcpu
+ i
];
131 topo
->drawer_id
= drawer
->id
;
132 topo
->book_id
= book
->id
;
133 topo
->socket_id
= socket
->id
;
134 topo
->core_id
= rcore
;
135 topo
->thread_id
= lcpu
+ i
;
136 topo
->dedicated
= tl_core
->d
;
137 cpumask_set_cpu(lcpu
+ i
, &drawer
->mask
);
138 cpumask_set_cpu(lcpu
+ i
, &book
->mask
);
139 cpumask_set_cpu(lcpu
+ i
, &socket
->mask
);
140 cpumask_set_cpu(lcpu
+ i
, &cpus_with_topology
);
141 smp_cpu_set_polarization(lcpu
+ i
, tl_core
->pp
);
146 static void clear_masks(void)
148 struct mask_info
*info
;
152 cpumask_clear(&info
->mask
);
157 cpumask_clear(&info
->mask
);
162 cpumask_clear(&info
->mask
);
167 static union topology_entry
*next_tle(union topology_entry
*tle
)
170 return (union topology_entry
*)((struct topology_core
*)tle
+ 1);
171 return (union topology_entry
*)((struct topology_container
*)tle
+ 1);
174 static void tl_to_masks(struct sysinfo_15_1_x
*info
)
176 struct mask_info
*socket
= &socket_info
;
177 struct mask_info
*book
= &book_info
;
178 struct mask_info
*drawer
= &drawer_info
;
179 union topology_entry
*tle
, *end
;
183 end
= (union topology_entry
*)((unsigned long)info
+ info
->length
);
187 drawer
= drawer
->next
;
188 drawer
->id
= tle
->container
.id
;
192 book
->id
= tle
->container
.id
;
195 socket
= socket
->next
;
196 socket
->id
= tle
->container
.id
;
199 add_cpus_to_mask(&tle
->cpu
, drawer
, book
, socket
);
209 static void topology_update_polarization_simple(void)
213 for_each_possible_cpu(cpu
)
214 smp_cpu_set_polarization(cpu
, POLARIZATION_HRZ
);
217 static int ptf(unsigned long fc
)
222 " .insn rre,0xb9a20000,%1,%1\n"
230 int topology_set_cpu_management(int fc
)
234 if (!MACHINE_HAS_TOPOLOGY
)
237 rc
= ptf(PTF_VERTICAL
);
239 rc
= ptf(PTF_HORIZONTAL
);
242 for_each_possible_cpu(cpu
)
243 smp_cpu_set_polarization(cpu
, POLARIZATION_UNKNOWN
);
247 static void update_cpu_masks(void)
249 struct cpu_topology_s390
*topo
;
252 for_each_possible_cpu(cpu
) {
253 topo
= &cpu_topology
[cpu
];
254 topo
->thread_mask
= cpu_thread_map(cpu
);
255 topo
->core_mask
= cpu_group_map(&socket_info
, cpu
);
256 topo
->book_mask
= cpu_group_map(&book_info
, cpu
);
257 topo
->drawer_mask
= cpu_group_map(&drawer_info
, cpu
);
258 if (topology_mode
!= TOPOLOGY_MODE_HW
) {
259 id
= topology_mode
== TOPOLOGY_MODE_PACKAGE
? 0 : cpu
;
260 topo
->thread_id
= cpu
;
262 topo
->socket_id
= id
;
264 topo
->drawer_id
= id
;
265 if (cpu_present(cpu
))
266 cpumask_set_cpu(cpu
, &cpus_with_topology
);
269 numa_update_cpu_topology();
272 void store_topology(struct sysinfo_15_1_x
*info
)
274 stsi(info
, 15, 1, topology_mnest_limit());
277 static void __arch_update_dedicated_flag(void *arg
)
279 if (topology_cpu_dedicated(smp_processor_id()))
280 set_cpu_flag(CIF_DEDICATED_CPU
);
282 clear_cpu_flag(CIF_DEDICATED_CPU
);
285 static int __arch_update_cpu_topology(void)
287 struct sysinfo_15_1_x
*info
= tl_info
;
290 mutex_lock(&smp_cpu_state_mutex
);
291 cpumask_clear(&cpus_with_topology
);
292 if (MACHINE_HAS_TOPOLOGY
) {
294 store_topology(info
);
298 if (!MACHINE_HAS_TOPOLOGY
)
299 topology_update_polarization_simple();
300 mutex_unlock(&smp_cpu_state_mutex
);
304 int arch_update_cpu_topology(void)
309 rc
= __arch_update_cpu_topology();
310 on_each_cpu(__arch_update_dedicated_flag
, NULL
, 0);
311 for_each_online_cpu(cpu
) {
312 dev
= get_cpu_device(cpu
);
313 kobject_uevent(&dev
->kobj
, KOBJ_CHANGE
);
318 static void topology_work_fn(struct work_struct
*work
)
320 rebuild_sched_domains();
323 void topology_schedule_update(void)
325 schedule_work(&topology_work
);
328 static void topology_flush_work(void)
330 flush_work(&topology_work
);
333 static void topology_timer_fn(struct timer_list
*unused
)
336 topology_schedule_update();
337 set_topology_timer();
340 static struct timer_list topology_timer
;
342 static atomic_t topology_poll
= ATOMIC_INIT(0);
344 static void set_topology_timer(void)
346 if (atomic_add_unless(&topology_poll
, -1, 0))
347 mod_timer(&topology_timer
, jiffies
+ HZ
/ 10);
349 mod_timer(&topology_timer
, jiffies
+ HZ
* 60);
352 void topology_expect_change(void)
354 if (!MACHINE_HAS_TOPOLOGY
)
356 /* This is racy, but it doesn't matter since it is just a heuristic.
357 * Worst case is that we poll in a higher frequency for a bit longer.
359 if (atomic_read(&topology_poll
) > 60)
361 atomic_add(60, &topology_poll
);
362 set_topology_timer();
365 static int cpu_management
;
367 static ssize_t
dispatching_show(struct device
*dev
,
368 struct device_attribute
*attr
,
373 mutex_lock(&smp_cpu_state_mutex
);
374 count
= sprintf(buf
, "%d\n", cpu_management
);
375 mutex_unlock(&smp_cpu_state_mutex
);
379 static ssize_t
dispatching_store(struct device
*dev
,
380 struct device_attribute
*attr
,
387 if (sscanf(buf
, "%d %c", &val
, &delim
) != 1)
389 if (val
!= 0 && val
!= 1)
393 mutex_lock(&smp_cpu_state_mutex
);
394 if (cpu_management
== val
)
396 rc
= topology_set_cpu_management(val
);
399 cpu_management
= val
;
400 topology_expect_change();
402 mutex_unlock(&smp_cpu_state_mutex
);
404 return rc
? rc
: count
;
406 static DEVICE_ATTR(dispatching
, 0644, dispatching_show
,
409 static ssize_t
cpu_polarization_show(struct device
*dev
,
410 struct device_attribute
*attr
, char *buf
)
415 mutex_lock(&smp_cpu_state_mutex
);
416 switch (smp_cpu_get_polarization(cpu
)) {
417 case POLARIZATION_HRZ
:
418 count
= sprintf(buf
, "horizontal\n");
420 case POLARIZATION_VL
:
421 count
= sprintf(buf
, "vertical:low\n");
423 case POLARIZATION_VM
:
424 count
= sprintf(buf
, "vertical:medium\n");
426 case POLARIZATION_VH
:
427 count
= sprintf(buf
, "vertical:high\n");
430 count
= sprintf(buf
, "unknown\n");
433 mutex_unlock(&smp_cpu_state_mutex
);
436 static DEVICE_ATTR(polarization
, 0444, cpu_polarization_show
, NULL
);
438 static struct attribute
*topology_cpu_attrs
[] = {
439 &dev_attr_polarization
.attr
,
443 static struct attribute_group topology_cpu_attr_group
= {
444 .attrs
= topology_cpu_attrs
,
447 static ssize_t
cpu_dedicated_show(struct device
*dev
,
448 struct device_attribute
*attr
, char *buf
)
453 mutex_lock(&smp_cpu_state_mutex
);
454 count
= sprintf(buf
, "%d\n", topology_cpu_dedicated(cpu
));
455 mutex_unlock(&smp_cpu_state_mutex
);
458 static DEVICE_ATTR(dedicated
, 0444, cpu_dedicated_show
, NULL
);
460 static struct attribute
*topology_extra_cpu_attrs
[] = {
461 &dev_attr_dedicated
.attr
,
465 static struct attribute_group topology_extra_cpu_attr_group
= {
466 .attrs
= topology_extra_cpu_attrs
,
469 int topology_cpu_init(struct cpu
*cpu
)
473 rc
= sysfs_create_group(&cpu
->dev
.kobj
, &topology_cpu_attr_group
);
474 if (rc
|| !MACHINE_HAS_TOPOLOGY
)
476 rc
= sysfs_create_group(&cpu
->dev
.kobj
, &topology_extra_cpu_attr_group
);
478 sysfs_remove_group(&cpu
->dev
.kobj
, &topology_cpu_attr_group
);
482 static const struct cpumask
*cpu_thread_mask(int cpu
)
484 return &cpu_topology
[cpu
].thread_mask
;
488 const struct cpumask
*cpu_coregroup_mask(int cpu
)
490 return &cpu_topology
[cpu
].core_mask
;
493 static const struct cpumask
*cpu_book_mask(int cpu
)
495 return &cpu_topology
[cpu
].book_mask
;
498 static const struct cpumask
*cpu_drawer_mask(int cpu
)
500 return &cpu_topology
[cpu
].drawer_mask
;
503 static struct sched_domain_topology_level s390_topology
[] = {
504 { cpu_thread_mask
, cpu_smt_flags
, SD_INIT_NAME(SMT
) },
505 { cpu_coregroup_mask
, cpu_core_flags
, SD_INIT_NAME(MC
) },
506 { cpu_book_mask
, SD_INIT_NAME(BOOK
) },
507 { cpu_drawer_mask
, SD_INIT_NAME(DRAWER
) },
508 { cpu_cpu_mask
, SD_INIT_NAME(DIE
) },
512 static void __init
alloc_masks(struct sysinfo_15_1_x
*info
,
513 struct mask_info
*mask
, int offset
)
517 nr_masks
= info
->mag
[TOPOLOGY_NR_MAG
- offset
];
518 for (i
= 0; i
< info
->mnest
- offset
; i
++)
519 nr_masks
*= info
->mag
[TOPOLOGY_NR_MAG
- offset
- 1 - i
];
520 nr_masks
= max(nr_masks
, 1);
521 for (i
= 0; i
< nr_masks
; i
++) {
522 mask
->next
= memblock_virt_alloc(sizeof(*mask
->next
), 8);
527 void __init
topology_init_early(void)
529 struct sysinfo_15_1_x
*info
;
531 set_sched_topology(s390_topology
);
532 if (topology_mode
== TOPOLOGY_MODE_UNINITIALIZED
) {
533 if (MACHINE_HAS_TOPOLOGY
)
534 topology_mode
= TOPOLOGY_MODE_HW
;
536 topology_mode
= TOPOLOGY_MODE_SINGLE
;
538 if (!MACHINE_HAS_TOPOLOGY
)
540 tl_info
= memblock_virt_alloc(PAGE_SIZE
, PAGE_SIZE
);
542 store_topology(info
);
543 pr_info("The CPU configuration topology of the machine is: %d %d %d %d %d %d / %d\n",
544 info
->mag
[0], info
->mag
[1], info
->mag
[2], info
->mag
[3],
545 info
->mag
[4], info
->mag
[5], info
->mnest
);
546 alloc_masks(info
, &socket_info
, 1);
547 alloc_masks(info
, &book_info
, 2);
548 alloc_masks(info
, &drawer_info
, 3);
550 __arch_update_cpu_topology();
551 __arch_update_dedicated_flag(NULL
);
554 static inline int topology_get_mode(int enabled
)
557 return TOPOLOGY_MODE_SINGLE
;
558 return MACHINE_HAS_TOPOLOGY
? TOPOLOGY_MODE_HW
: TOPOLOGY_MODE_PACKAGE
;
561 static inline int topology_is_enabled(void)
563 return topology_mode
!= TOPOLOGY_MODE_SINGLE
;
566 static int __init
topology_setup(char *str
)
571 rc
= kstrtobool(str
, &enabled
);
574 topology_mode
= topology_get_mode(enabled
);
577 early_param("topology", topology_setup
);
579 static int topology_ctl_handler(struct ctl_table
*ctl
, int write
,
580 void __user
*buffer
, size_t *lenp
, loff_t
*ppos
)
586 if (!*lenp
|| *ppos
) {
591 strncpy(buf
, topology_is_enabled() ? "1\n" : "0\n",
593 len
= strnlen(buf
, ARRAY_SIZE(buf
));
596 if (copy_to_user(buffer
, buf
, len
))
601 if (copy_from_user(buf
, buffer
, len
> sizeof(buf
) ? sizeof(buf
) : len
))
603 if (buf
[0] != '0' && buf
[0] != '1')
605 mutex_lock(&smp_cpu_state_mutex
);
606 new_mode
= topology_get_mode(buf
[0] == '1');
607 if (topology_mode
!= new_mode
) {
608 topology_mode
= new_mode
;
609 topology_schedule_update();
611 mutex_unlock(&smp_cpu_state_mutex
);
612 topology_flush_work();
619 static struct ctl_table topology_ctl_table
[] = {
621 .procname
= "topology",
623 .proc_handler
= topology_ctl_handler
,
628 static struct ctl_table topology_dir_table
[] = {
633 .child
= topology_ctl_table
,
638 static int __init
topology_init(void)
640 timer_setup(&topology_timer
, topology_timer_fn
, TIMER_DEFERRABLE
);
641 if (MACHINE_HAS_TOPOLOGY
)
642 set_topology_timer();
644 topology_update_polarization_simple();
645 register_sysctl_table(topology_dir_table
);
646 return device_create_file(cpu_subsys
.dev_root
, &dev_attr_dispatching
);
648 device_initcall(topology_init
);