1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright IBM Corp. 2007, 2011
4 * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>
7 #define KMSG_COMPONENT "cpu"
8 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
10 #include <linux/workqueue.h>
11 #include <linux/memblock.h>
12 #include <linux/uaccess.h>
13 #include <linux/sysctl.h>
14 #include <linux/cpuset.h>
15 #include <linux/device.h>
16 #include <linux/export.h>
17 #include <linux/kernel.h>
18 #include <linux/sched.h>
19 #include <linux/sched/topology.h>
20 #include <linux/delay.h>
21 #include <linux/init.h>
22 #include <linux/slab.h>
23 #include <linux/cpu.h>
24 #include <linux/smp.h>
26 #include <linux/nodemask.h>
27 #include <linux/node.h>
28 #include <asm/sysinfo.h>
31 #define PTF_HORIZONTAL (0UL)
32 #define PTF_VERTICAL (1UL)
33 #define PTF_CHECK (2UL)
38 TOPOLOGY_MODE_PACKAGE
,
39 TOPOLOGY_MODE_UNINITIALIZED
43 struct mask_info
*next
;
48 static int topology_mode
= TOPOLOGY_MODE_UNINITIALIZED
;
49 static void set_topology_timer(void);
50 static void topology_work_fn(struct work_struct
*work
);
51 static struct sysinfo_15_1_x
*tl_info
;
53 static DECLARE_WORK(topology_work
, topology_work_fn
);
56 * Socket/Book linked lists and cpu_topology updates are
57 * protected by "sched_domains_mutex".
59 static struct mask_info socket_info
;
60 static struct mask_info book_info
;
61 static struct mask_info drawer_info
;
63 struct cpu_topology_s390 cpu_topology
[NR_CPUS
];
64 EXPORT_SYMBOL_GPL(cpu_topology
);
66 cpumask_t cpus_with_topology
;
68 int __cpu_to_node(int cpu
)
70 return cpu_topology
[cpu
].node_id
;
73 EXPORT_SYMBOL(__cpu_to_node
);
75 static cpumask_t
cpu_group_map(struct mask_info
*info
, unsigned int cpu
)
79 cpumask_copy(&mask
, cpumask_of(cpu
));
80 switch (topology_mode
) {
81 case TOPOLOGY_MODE_HW
:
83 if (cpumask_test_cpu(cpu
, &info
->mask
)) {
89 if (cpumask_empty(&mask
))
90 cpumask_copy(&mask
, cpumask_of(cpu
));
92 case TOPOLOGY_MODE_PACKAGE
:
93 cpumask_copy(&mask
, cpu_present_mask
);
97 case TOPOLOGY_MODE_SINGLE
:
98 cpumask_copy(&mask
, cpumask_of(cpu
));
104 static cpumask_t
cpu_thread_map(unsigned int cpu
)
109 cpumask_copy(&mask
, cpumask_of(cpu
));
110 if (topology_mode
!= TOPOLOGY_MODE_HW
)
112 cpu
-= cpu
% (smp_cpu_mtid
+ 1);
113 for (i
= 0; i
<= smp_cpu_mtid
; i
++)
114 if (cpu_present(cpu
+ i
))
115 cpumask_set_cpu(cpu
+ i
, &mask
);
119 #define TOPOLOGY_CORE_BITS 64
121 static void add_cpus_to_mask(struct topology_core
*tl_core
,
122 struct mask_info
*drawer
,
123 struct mask_info
*book
,
124 struct mask_info
*socket
)
126 struct cpu_topology_s390
*topo
;
129 for_each_set_bit(core
, &tl_core
->mask
, TOPOLOGY_CORE_BITS
) {
133 rcore
= TOPOLOGY_CORE_BITS
- 1 - core
+ tl_core
->origin
;
134 lcpu
= smp_find_processor_id(rcore
<< smp_cpu_mt_shift
);
137 for (i
= 0; i
<= smp_cpu_mtid
; i
++) {
138 topo
= &cpu_topology
[lcpu
+ i
];
139 topo
->drawer_id
= drawer
->id
;
140 topo
->book_id
= book
->id
;
141 topo
->socket_id
= socket
->id
;
142 topo
->core_id
= rcore
;
143 topo
->thread_id
= lcpu
+ i
;
144 topo
->dedicated
= tl_core
->d
;
145 cpumask_set_cpu(lcpu
+ i
, &drawer
->mask
);
146 cpumask_set_cpu(lcpu
+ i
, &book
->mask
);
147 cpumask_set_cpu(lcpu
+ i
, &socket
->mask
);
148 cpumask_set_cpu(lcpu
+ i
, &cpus_with_topology
);
149 smp_cpu_set_polarization(lcpu
+ i
, tl_core
->pp
);
154 static void clear_masks(void)
156 struct mask_info
*info
;
160 cpumask_clear(&info
->mask
);
165 cpumask_clear(&info
->mask
);
170 cpumask_clear(&info
->mask
);
175 static union topology_entry
*next_tle(union topology_entry
*tle
)
178 return (union topology_entry
*)((struct topology_core
*)tle
+ 1);
179 return (union topology_entry
*)((struct topology_container
*)tle
+ 1);
182 static void tl_to_masks(struct sysinfo_15_1_x
*info
)
184 struct mask_info
*socket
= &socket_info
;
185 struct mask_info
*book
= &book_info
;
186 struct mask_info
*drawer
= &drawer_info
;
187 union topology_entry
*tle
, *end
;
191 end
= (union topology_entry
*)((unsigned long)info
+ info
->length
);
195 drawer
= drawer
->next
;
196 drawer
->id
= tle
->container
.id
;
200 book
->id
= tle
->container
.id
;
203 socket
= socket
->next
;
204 socket
->id
= tle
->container
.id
;
207 add_cpus_to_mask(&tle
->cpu
, drawer
, book
, socket
);
217 static void topology_update_polarization_simple(void)
221 for_each_possible_cpu(cpu
)
222 smp_cpu_set_polarization(cpu
, POLARIZATION_HRZ
);
225 static int ptf(unsigned long fc
)
230 " .insn rre,0xb9a20000,%1,%1\n"
238 int topology_set_cpu_management(int fc
)
242 if (!MACHINE_HAS_TOPOLOGY
)
245 rc
= ptf(PTF_VERTICAL
);
247 rc
= ptf(PTF_HORIZONTAL
);
250 for_each_possible_cpu(cpu
)
251 smp_cpu_set_polarization(cpu
, POLARIZATION_UNKNOWN
);
255 static void update_cpu_masks(void)
257 struct cpu_topology_s390
*topo
;
260 for_each_possible_cpu(cpu
) {
261 topo
= &cpu_topology
[cpu
];
262 topo
->thread_mask
= cpu_thread_map(cpu
);
263 topo
->core_mask
= cpu_group_map(&socket_info
, cpu
);
264 topo
->book_mask
= cpu_group_map(&book_info
, cpu
);
265 topo
->drawer_mask
= cpu_group_map(&drawer_info
, cpu
);
266 if (topology_mode
!= TOPOLOGY_MODE_HW
) {
267 id
= topology_mode
== TOPOLOGY_MODE_PACKAGE
? 0 : cpu
;
268 topo
->thread_id
= cpu
;
270 topo
->socket_id
= id
;
272 topo
->drawer_id
= id
;
273 if (cpu_present(cpu
))
274 cpumask_set_cpu(cpu
, &cpus_with_topology
);
277 numa_update_cpu_topology();
280 void store_topology(struct sysinfo_15_1_x
*info
)
282 stsi(info
, 15, 1, topology_mnest_limit());
285 static void __arch_update_dedicated_flag(void *arg
)
287 if (topology_cpu_dedicated(smp_processor_id()))
288 set_cpu_flag(CIF_DEDICATED_CPU
);
290 clear_cpu_flag(CIF_DEDICATED_CPU
);
293 static int __arch_update_cpu_topology(void)
295 struct sysinfo_15_1_x
*info
= tl_info
;
298 mutex_lock(&smp_cpu_state_mutex
);
299 cpumask_clear(&cpus_with_topology
);
300 if (MACHINE_HAS_TOPOLOGY
) {
302 store_topology(info
);
306 if (!MACHINE_HAS_TOPOLOGY
)
307 topology_update_polarization_simple();
308 mutex_unlock(&smp_cpu_state_mutex
);
312 int arch_update_cpu_topology(void)
317 rc
= __arch_update_cpu_topology();
318 on_each_cpu(__arch_update_dedicated_flag
, NULL
, 0);
319 for_each_online_cpu(cpu
) {
320 dev
= get_cpu_device(cpu
);
321 kobject_uevent(&dev
->kobj
, KOBJ_CHANGE
);
326 static void topology_work_fn(struct work_struct
*work
)
328 rebuild_sched_domains();
331 void topology_schedule_update(void)
333 schedule_work(&topology_work
);
336 static void topology_flush_work(void)
338 flush_work(&topology_work
);
341 static void topology_timer_fn(struct timer_list
*unused
)
344 topology_schedule_update();
345 set_topology_timer();
348 static struct timer_list topology_timer
;
350 static atomic_t topology_poll
= ATOMIC_INIT(0);
352 static void set_topology_timer(void)
354 if (atomic_add_unless(&topology_poll
, -1, 0))
355 mod_timer(&topology_timer
, jiffies
+ HZ
/ 10);
357 mod_timer(&topology_timer
, jiffies
+ HZ
* 60);
360 void topology_expect_change(void)
362 if (!MACHINE_HAS_TOPOLOGY
)
364 /* This is racy, but it doesn't matter since it is just a heuristic.
365 * Worst case is that we poll in a higher frequency for a bit longer.
367 if (atomic_read(&topology_poll
) > 60)
369 atomic_add(60, &topology_poll
);
370 set_topology_timer();
373 static int cpu_management
;
375 static ssize_t
dispatching_show(struct device
*dev
,
376 struct device_attribute
*attr
,
381 mutex_lock(&smp_cpu_state_mutex
);
382 count
= sprintf(buf
, "%d\n", cpu_management
);
383 mutex_unlock(&smp_cpu_state_mutex
);
387 static ssize_t
dispatching_store(struct device
*dev
,
388 struct device_attribute
*attr
,
395 if (sscanf(buf
, "%d %c", &val
, &delim
) != 1)
397 if (val
!= 0 && val
!= 1)
401 mutex_lock(&smp_cpu_state_mutex
);
402 if (cpu_management
== val
)
404 rc
= topology_set_cpu_management(val
);
407 cpu_management
= val
;
408 topology_expect_change();
410 mutex_unlock(&smp_cpu_state_mutex
);
412 return rc
? rc
: count
;
414 static DEVICE_ATTR_RW(dispatching
);
416 static ssize_t
cpu_polarization_show(struct device
*dev
,
417 struct device_attribute
*attr
, char *buf
)
422 mutex_lock(&smp_cpu_state_mutex
);
423 switch (smp_cpu_get_polarization(cpu
)) {
424 case POLARIZATION_HRZ
:
425 count
= sprintf(buf
, "horizontal\n");
427 case POLARIZATION_VL
:
428 count
= sprintf(buf
, "vertical:low\n");
430 case POLARIZATION_VM
:
431 count
= sprintf(buf
, "vertical:medium\n");
433 case POLARIZATION_VH
:
434 count
= sprintf(buf
, "vertical:high\n");
437 count
= sprintf(buf
, "unknown\n");
440 mutex_unlock(&smp_cpu_state_mutex
);
443 static DEVICE_ATTR(polarization
, 0444, cpu_polarization_show
, NULL
);
445 static struct attribute
*topology_cpu_attrs
[] = {
446 &dev_attr_polarization
.attr
,
450 static struct attribute_group topology_cpu_attr_group
= {
451 .attrs
= topology_cpu_attrs
,
454 static ssize_t
cpu_dedicated_show(struct device
*dev
,
455 struct device_attribute
*attr
, char *buf
)
460 mutex_lock(&smp_cpu_state_mutex
);
461 count
= sprintf(buf
, "%d\n", topology_cpu_dedicated(cpu
));
462 mutex_unlock(&smp_cpu_state_mutex
);
465 static DEVICE_ATTR(dedicated
, 0444, cpu_dedicated_show
, NULL
);
467 static struct attribute
*topology_extra_cpu_attrs
[] = {
468 &dev_attr_dedicated
.attr
,
472 static struct attribute_group topology_extra_cpu_attr_group
= {
473 .attrs
= topology_extra_cpu_attrs
,
476 int topology_cpu_init(struct cpu
*cpu
)
480 rc
= sysfs_create_group(&cpu
->dev
.kobj
, &topology_cpu_attr_group
);
481 if (rc
|| !MACHINE_HAS_TOPOLOGY
)
483 rc
= sysfs_create_group(&cpu
->dev
.kobj
, &topology_extra_cpu_attr_group
);
485 sysfs_remove_group(&cpu
->dev
.kobj
, &topology_cpu_attr_group
);
489 static const struct cpumask
*cpu_thread_mask(int cpu
)
491 return &cpu_topology
[cpu
].thread_mask
;
495 const struct cpumask
*cpu_coregroup_mask(int cpu
)
497 return &cpu_topology
[cpu
].core_mask
;
500 static const struct cpumask
*cpu_book_mask(int cpu
)
502 return &cpu_topology
[cpu
].book_mask
;
505 static const struct cpumask
*cpu_drawer_mask(int cpu
)
507 return &cpu_topology
[cpu
].drawer_mask
;
510 static struct sched_domain_topology_level s390_topology
[] = {
511 { cpu_thread_mask
, cpu_smt_flags
, SD_INIT_NAME(SMT
) },
512 { cpu_coregroup_mask
, cpu_core_flags
, SD_INIT_NAME(MC
) },
513 { cpu_book_mask
, SD_INIT_NAME(BOOK
) },
514 { cpu_drawer_mask
, SD_INIT_NAME(DRAWER
) },
515 { cpu_cpu_mask
, SD_INIT_NAME(DIE
) },
519 static void __init
alloc_masks(struct sysinfo_15_1_x
*info
,
520 struct mask_info
*mask
, int offset
)
524 nr_masks
= info
->mag
[TOPOLOGY_NR_MAG
- offset
];
525 for (i
= 0; i
< info
->mnest
- offset
; i
++)
526 nr_masks
*= info
->mag
[TOPOLOGY_NR_MAG
- offset
- 1 - i
];
527 nr_masks
= max(nr_masks
, 1);
528 for (i
= 0; i
< nr_masks
; i
++) {
529 mask
->next
= memblock_alloc(sizeof(*mask
->next
), 8);
531 panic("%s: Failed to allocate %zu bytes align=0x%x\n",
532 __func__
, sizeof(*mask
->next
), 8);
537 void __init
topology_init_early(void)
539 struct sysinfo_15_1_x
*info
;
541 set_sched_topology(s390_topology
);
542 if (topology_mode
== TOPOLOGY_MODE_UNINITIALIZED
) {
543 if (MACHINE_HAS_TOPOLOGY
)
544 topology_mode
= TOPOLOGY_MODE_HW
;
546 topology_mode
= TOPOLOGY_MODE_SINGLE
;
548 if (!MACHINE_HAS_TOPOLOGY
)
550 tl_info
= memblock_alloc(PAGE_SIZE
, PAGE_SIZE
);
552 panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
553 __func__
, PAGE_SIZE
, PAGE_SIZE
);
555 store_topology(info
);
556 pr_info("The CPU configuration topology of the machine is: %d %d %d %d %d %d / %d\n",
557 info
->mag
[0], info
->mag
[1], info
->mag
[2], info
->mag
[3],
558 info
->mag
[4], info
->mag
[5], info
->mnest
);
559 alloc_masks(info
, &socket_info
, 1);
560 alloc_masks(info
, &book_info
, 2);
561 alloc_masks(info
, &drawer_info
, 3);
563 __arch_update_cpu_topology();
564 __arch_update_dedicated_flag(NULL
);
567 static inline int topology_get_mode(int enabled
)
570 return TOPOLOGY_MODE_SINGLE
;
571 return MACHINE_HAS_TOPOLOGY
? TOPOLOGY_MODE_HW
: TOPOLOGY_MODE_PACKAGE
;
574 static inline int topology_is_enabled(void)
576 return topology_mode
!= TOPOLOGY_MODE_SINGLE
;
579 static int __init
topology_setup(char *str
)
584 rc
= kstrtobool(str
, &enabled
);
587 topology_mode
= topology_get_mode(enabled
);
590 early_param("topology", topology_setup
);
592 static int topology_ctl_handler(struct ctl_table
*ctl
, int write
,
593 void __user
*buffer
, size_t *lenp
, loff_t
*ppos
)
595 int enabled
= topology_is_enabled();
598 struct ctl_table ctl_entry
= {
599 .procname
= ctl
->procname
,
601 .maxlen
= sizeof(int),
602 .extra1
= SYSCTL_ZERO
,
603 .extra2
= SYSCTL_ONE
,
606 rc
= proc_douintvec_minmax(&ctl_entry
, write
, buffer
, lenp
, ppos
);
607 if (rc
< 0 || !write
)
610 mutex_lock(&smp_cpu_state_mutex
);
611 new_mode
= topology_get_mode(enabled
);
612 if (topology_mode
!= new_mode
) {
613 topology_mode
= new_mode
;
614 topology_schedule_update();
616 mutex_unlock(&smp_cpu_state_mutex
);
617 topology_flush_work();
622 static struct ctl_table topology_ctl_table
[] = {
624 .procname
= "topology",
626 .proc_handler
= topology_ctl_handler
,
631 static struct ctl_table topology_dir_table
[] = {
636 .child
= topology_ctl_table
,
641 static int __init
topology_init(void)
643 timer_setup(&topology_timer
, topology_timer_fn
, TIMER_DEFERRABLE
);
644 if (MACHINE_HAS_TOPOLOGY
)
645 set_topology_timer();
647 topology_update_polarization_simple();
648 register_sysctl_table(topology_dir_table
);
649 return device_create_file(cpu_subsys
.dev_root
, &dev_attr_dispatching
);
651 device_initcall(topology_init
);