]>
Commit | Line | Data |
---|---|---|
dbd70fb4 | 1 | /* |
a53c8fab | 2 | * Copyright IBM Corp. 2007, 2011 |
dbd70fb4 HC |
3 | * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com> |
4 | */ | |
5 | ||
395d31d4 MS |
6 | #define KMSG_COMPONENT "cpu" |
7 | #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt | |
8 | ||
83a24e32 | 9 | #include <linux/workqueue.h> |
83a24e32 HC |
10 | #include <linux/cpuset.h> |
11 | #include <linux/device.h> | |
80020fbd | 12 | #include <linux/export.h> |
83a24e32 | 13 | #include <linux/kernel.h> |
dbd70fb4 | 14 | #include <linux/sched.h> |
83a24e32 | 15 | #include <linux/delay.h> |
d05d15da HC |
16 | #include <linux/init.h> |
17 | #include <linux/slab.h> | |
dbd70fb4 HC |
18 | #include <linux/cpu.h> |
19 | #include <linux/smp.h> | |
83a24e32 | 20 | #include <linux/mm.h> |
3a368f74 PH |
21 | #include <linux/nodemask.h> |
22 | #include <linux/node.h> | |
78609132 | 23 | #include <asm/sysinfo.h> |
3a368f74 | 24 | #include <asm/numa.h> |
dbd70fb4 | 25 | |
c10fde0d HC |
26 | #define PTF_HORIZONTAL (0UL) |
27 | #define PTF_VERTICAL (1UL) | |
28 | #define PTF_CHECK (2UL) | |
dbd70fb4 | 29 | |
4cb14bc8 HC |
30 | struct mask_info { |
31 | struct mask_info *next; | |
10d38589 | 32 | unsigned char id; |
dbd70fb4 HC |
33 | cpumask_t mask; |
34 | }; | |
35 | ||
d1e57508 | 36 | static void set_topology_timer(void); |
dbd70fb4 | 37 | static void topology_work_fn(struct work_struct *work); |
c30f91b6 | 38 | static struct sysinfo_15_1_x *tl_info; |
dbd70fb4 | 39 | |
d1e57508 HC |
40 | static int topology_enabled = 1; |
41 | static DECLARE_WORK(topology_work, topology_work_fn); | |
d00aa4e7 | 42 | |
3a3814c2 MH |
43 | /* |
44 | * Socket/Book linked lists and per_cpu(cpu_topology) updates are | |
45 | * protected by "sched_domains_mutex". | |
46 | */ | |
d1e57508 | 47 | static struct mask_info socket_info; |
4cb14bc8 | 48 | static struct mask_info book_info; |
d1e57508 | 49 | |
da0c636e HC |
50 | DEFINE_PER_CPU(struct cpu_topology_s390, cpu_topology); |
51 | EXPORT_PER_CPU_SYMBOL_GPL(cpu_topology); | |
83a24e32 | 52 | |
4cb14bc8 | 53 | static cpumask_t cpu_group_map(struct mask_info *info, unsigned int cpu) |
dbd70fb4 | 54 | { |
dbd70fb4 HC |
55 | cpumask_t mask; |
56 | ||
d1e57508 HC |
57 | cpumask_copy(&mask, cpumask_of(cpu)); |
58 | if (!topology_enabled || !MACHINE_HAS_TOPOLOGY) | |
0b52783d | 59 | return mask; |
d1e57508 HC |
60 | for (; info; info = info->next) { |
61 | if (cpumask_test_cpu(cpu, &info->mask)) | |
62 | return info->mask; | |
0b52783d | 63 | } |
dbd70fb4 HC |
64 | return mask; |
65 | } | |
66 | ||
10ad34bc MS |
67 | static cpumask_t cpu_thread_map(unsigned int cpu) |
68 | { | |
69 | cpumask_t mask; | |
70 | int i; | |
71 | ||
72 | cpumask_copy(&mask, cpumask_of(cpu)); | |
73 | if (!topology_enabled || !MACHINE_HAS_TOPOLOGY) | |
74 | return mask; | |
75 | cpu -= cpu % (smp_cpu_mtid + 1); | |
76 | for (i = 0; i <= smp_cpu_mtid; i++) | |
77 | if (cpu_present(cpu + i)) | |
78 | cpumask_set_cpu(cpu + i, &mask); | |
79 | return mask; | |
80 | } | |
81 | ||
82 | static struct mask_info *add_cpus_to_mask(struct topology_core *tl_core, | |
f6bf1a8a | 83 | struct mask_info *book, |
d1e57508 HC |
84 | struct mask_info *socket, |
85 | int one_socket_per_cpu) | |
dbd70fb4 | 86 | { |
10ad34bc | 87 | unsigned int core; |
dbd70fb4 | 88 | |
10ad34bc MS |
89 | for_each_set_bit(core, &tl_core->mask[0], TOPOLOGY_CORE_BITS) { |
90 | unsigned int rcore; | |
91 | int lcpu, i; | |
dbd70fb4 | 92 | |
10ad34bc MS |
93 | rcore = TOPOLOGY_CORE_BITS - 1 - core + tl_core->origin; |
94 | lcpu = smp_find_processor_id(rcore << smp_cpu_mt_shift); | |
d1e57508 HC |
95 | if (lcpu < 0) |
96 | continue; | |
10ad34bc | 97 | for (i = 0; i <= smp_cpu_mtid; i++) { |
da0c636e HC |
98 | per_cpu(cpu_topology, lcpu + i).book_id = book->id; |
99 | per_cpu(cpu_topology, lcpu + i).core_id = rcore; | |
100 | per_cpu(cpu_topology, lcpu + i).thread_id = lcpu + i; | |
10ad34bc MS |
101 | cpumask_set_cpu(lcpu + i, &book->mask); |
102 | cpumask_set_cpu(lcpu + i, &socket->mask); | |
103 | if (one_socket_per_cpu) | |
da0c636e | 104 | per_cpu(cpu_topology, lcpu + i).socket_id = rcore; |
10ad34bc | 105 | else |
da0c636e | 106 | per_cpu(cpu_topology, lcpu + i).socket_id = socket->id; |
10ad34bc | 107 | smp_cpu_set_polarization(lcpu + i, tl_core->pp); |
dbd70fb4 | 108 | } |
10ad34bc MS |
109 | if (one_socket_per_cpu) |
110 | socket = socket->next; | |
dbd70fb4 | 111 | } |
d1e57508 | 112 | return socket; |
dbd70fb4 HC |
113 | } |
114 | ||
4cb14bc8 | 115 | static void clear_masks(void) |
dbd70fb4 | 116 | { |
4cb14bc8 | 117 | struct mask_info *info; |
dbd70fb4 | 118 | |
d1e57508 | 119 | info = &socket_info; |
4cb14bc8 | 120 | while (info) { |
0f1959f5 | 121 | cpumask_clear(&info->mask); |
4cb14bc8 HC |
122 | info = info->next; |
123 | } | |
4cb14bc8 HC |
124 | info = &book_info; |
125 | while (info) { | |
0f1959f5 | 126 | cpumask_clear(&info->mask); |
4cb14bc8 | 127 | info = info->next; |
dbd70fb4 HC |
128 | } |
129 | } | |
130 | ||
c30f91b6 | 131 | static union topology_entry *next_tle(union topology_entry *tle) |
dbd70fb4 | 132 | { |
c30f91b6 | 133 | if (!tle->nl) |
10ad34bc | 134 | return (union topology_entry *)((struct topology_core *)tle + 1); |
c30f91b6 | 135 | return (union topology_entry *)((struct topology_container *)tle + 1); |
dbd70fb4 HC |
136 | } |
137 | ||
d1e57508 | 138 | static void __tl_to_masks_generic(struct sysinfo_15_1_x *info) |
dbd70fb4 | 139 | { |
d1e57508 | 140 | struct mask_info *socket = &socket_info; |
83a24e32 | 141 | struct mask_info *book = &book_info; |
c30f91b6 | 142 | union topology_entry *tle, *end; |
4cb14bc8 | 143 | |
c10fde0d | 144 | tle = info->tle; |
c30f91b6 | 145 | end = (union topology_entry *)((unsigned long)info + info->length); |
dbd70fb4 HC |
146 | while (tle < end) { |
147 | switch (tle->nl) { | |
dbd70fb4 | 148 | case 2: |
4cb14bc8 HC |
149 | book = book->next; |
150 | book->id = tle->container.id; | |
dbd70fb4 HC |
151 | break; |
152 | case 1: | |
d1e57508 HC |
153 | socket = socket->next; |
154 | socket->id = tle->container.id; | |
dbd70fb4 HC |
155 | break; |
156 | case 0: | |
d1e57508 | 157 | add_cpus_to_mask(&tle->cpu, book, socket, 0); |
dbd70fb4 HC |
158 | break; |
159 | default: | |
4cb14bc8 | 160 | clear_masks(); |
4baeb964 | 161 | return; |
dbd70fb4 HC |
162 | } |
163 | tle = next_tle(tle); | |
164 | } | |
4baeb964 HC |
165 | } |
166 | ||
d1e57508 | 167 | static void __tl_to_masks_z10(struct sysinfo_15_1_x *info) |
4baeb964 | 168 | { |
d1e57508 | 169 | struct mask_info *socket = &socket_info; |
4baeb964 HC |
170 | struct mask_info *book = &book_info; |
171 | union topology_entry *tle, *end; | |
172 | ||
173 | tle = info->tle; | |
174 | end = (union topology_entry *)((unsigned long)info + info->length); | |
175 | while (tle < end) { | |
176 | switch (tle->nl) { | |
177 | case 1: | |
178 | book = book->next; | |
179 | book->id = tle->container.id; | |
180 | break; | |
181 | case 0: | |
d1e57508 | 182 | socket = add_cpus_to_mask(&tle->cpu, book, socket, 1); |
4baeb964 HC |
183 | break; |
184 | default: | |
185 | clear_masks(); | |
186 | return; | |
187 | } | |
188 | tle = next_tle(tle); | |
189 | } | |
190 | } | |
191 | ||
d1e57508 | 192 | static void tl_to_masks(struct sysinfo_15_1_x *info) |
4baeb964 HC |
193 | { |
194 | struct cpuid cpu_id; | |
195 | ||
d1e57508 | 196 | get_cpu_id(&cpu_id); |
4baeb964 HC |
197 | clear_masks(); |
198 | switch (cpu_id.machine) { | |
199 | case 0x2097: | |
200 | case 0x2098: | |
d1e57508 | 201 | __tl_to_masks_z10(info); |
4baeb964 HC |
202 | break; |
203 | default: | |
d1e57508 | 204 | __tl_to_masks_generic(info); |
4baeb964 | 205 | } |
dbd70fb4 HC |
206 | } |
207 | ||
c10fde0d HC |
208 | static void topology_update_polarization_simple(void) |
209 | { | |
210 | int cpu; | |
211 | ||
212 | mutex_lock(&smp_cpu_state_mutex); | |
5439050f | 213 | for_each_possible_cpu(cpu) |
50ab9a9a | 214 | smp_cpu_set_polarization(cpu, POLARIZATION_HRZ); |
c10fde0d HC |
215 | mutex_unlock(&smp_cpu_state_mutex); |
216 | } | |
217 | ||
218 | static int ptf(unsigned long fc) | |
dbd70fb4 HC |
219 | { |
220 | int rc; | |
221 | ||
222 | asm volatile( | |
223 | " .insn rre,0xb9a20000,%1,%1\n" | |
224 | " ipm %0\n" | |
225 | " srl %0,28\n" | |
226 | : "=d" (rc) | |
c10fde0d HC |
227 | : "d" (fc) : "cc"); |
228 | return rc; | |
229 | } | |
230 | ||
231 | int topology_set_cpu_management(int fc) | |
232 | { | |
83a24e32 | 233 | int cpu, rc; |
c10fde0d | 234 | |
9186d7a9 | 235 | if (!MACHINE_HAS_TOPOLOGY) |
c10fde0d HC |
236 | return -EOPNOTSUPP; |
237 | if (fc) | |
238 | rc = ptf(PTF_VERTICAL); | |
239 | else | |
240 | rc = ptf(PTF_HORIZONTAL); | |
241 | if (rc) | |
242 | return -EBUSY; | |
5439050f | 243 | for_each_possible_cpu(cpu) |
50ab9a9a | 244 | smp_cpu_set_polarization(cpu, POLARIZATION_UNKNOWN); |
dbd70fb4 HC |
245 | return rc; |
246 | } | |
247 | ||
d1e57508 | 248 | static void update_cpu_masks(void) |
d00aa4e7 HC |
249 | { |
250 | int cpu; | |
251 | ||
4cb14bc8 | 252 | for_each_possible_cpu(cpu) { |
da0c636e HC |
253 | per_cpu(cpu_topology, cpu).thread_mask = cpu_thread_map(cpu); |
254 | per_cpu(cpu_topology, cpu).core_mask = cpu_group_map(&socket_info, cpu); | |
255 | per_cpu(cpu_topology, cpu).book_mask = cpu_group_map(&book_info, cpu); | |
d1e57508 | 256 | if (!MACHINE_HAS_TOPOLOGY) { |
da0c636e HC |
257 | per_cpu(cpu_topology, cpu).thread_id = cpu; |
258 | per_cpu(cpu_topology, cpu).core_id = cpu; | |
259 | per_cpu(cpu_topology, cpu).socket_id = cpu; | |
260 | per_cpu(cpu_topology, cpu).book_id = cpu; | |
d1e57508 | 261 | } |
4cb14bc8 | 262 | } |
3a368f74 | 263 | numa_update_cpu_topology(); |
4cb14bc8 HC |
264 | } |
265 | ||
96f4a70d | 266 | void store_topology(struct sysinfo_15_1_x *info) |
4cb14bc8 | 267 | { |
fade4dc4 HC |
268 | if (topology_max_mnest >= 3) |
269 | stsi(info, 15, 1, 3); | |
270 | else | |
271 | stsi(info, 15, 1, 2); | |
d00aa4e7 HC |
272 | } |
273 | ||
ee79d1bd | 274 | int arch_update_cpu_topology(void) |
dbd70fb4 | 275 | { |
c30f91b6 | 276 | struct sysinfo_15_1_x *info = tl_info; |
8a25a2fd | 277 | struct device *dev; |
3a368f74 | 278 | int cpu, rc = 0; |
dbd70fb4 | 279 | |
3a368f74 PH |
280 | if (MACHINE_HAS_TOPOLOGY) { |
281 | rc = 1; | |
282 | store_topology(info); | |
283 | tl_to_masks(info); | |
c10fde0d | 284 | } |
d1e57508 | 285 | update_cpu_masks(); |
3a368f74 PH |
286 | if (!MACHINE_HAS_TOPOLOGY) |
287 | topology_update_polarization_simple(); | |
dbd70fb4 | 288 | for_each_online_cpu(cpu) { |
8a25a2fd KS |
289 | dev = get_cpu_device(cpu); |
290 | kobject_uevent(&dev->kobj, KOBJ_CHANGE); | |
dbd70fb4 | 291 | } |
3a368f74 | 292 | return rc; |
dbd70fb4 HC |
293 | } |
294 | ||
fd781fa2 HC |
295 | static void topology_work_fn(struct work_struct *work) |
296 | { | |
f414f5f1 | 297 | rebuild_sched_domains(); |
dbd70fb4 HC |
298 | } |
299 | ||
c10fde0d HC |
300 | void topology_schedule_update(void) |
301 | { | |
302 | schedule_work(&topology_work); | |
303 | } | |
304 | ||
dbd70fb4 HC |
305 | static void topology_timer_fn(unsigned long ignored) |
306 | { | |
c10fde0d HC |
307 | if (ptf(PTF_CHECK)) |
308 | topology_schedule_update(); | |
dbd70fb4 HC |
309 | set_topology_timer(); |
310 | } | |
311 | ||
d68bddb7 HC |
312 | static struct timer_list topology_timer = |
313 | TIMER_DEFERRED_INITIALIZER(topology_timer_fn, 0, 0); | |
314 | ||
315 | static atomic_t topology_poll = ATOMIC_INIT(0); | |
316 | ||
dbd70fb4 HC |
317 | static void set_topology_timer(void) |
318 | { | |
d68bddb7 HC |
319 | if (atomic_add_unless(&topology_poll, -1, 0)) |
320 | mod_timer(&topology_timer, jiffies + HZ / 10); | |
321 | else | |
322 | mod_timer(&topology_timer, jiffies + HZ * 60); | |
323 | } | |
324 | ||
325 | void topology_expect_change(void) | |
326 | { | |
327 | if (!MACHINE_HAS_TOPOLOGY) | |
328 | return; | |
329 | /* This is racy, but it doesn't matter since it is just a heuristic. | |
330 | * Worst case is that we poll in a higher frequency for a bit longer. | |
331 | */ | |
332 | if (atomic_read(&topology_poll) > 60) | |
333 | return; | |
334 | atomic_add(60, &topology_poll); | |
335 | set_topology_timer(); | |
dbd70fb4 HC |
336 | } |
337 | ||
83a24e32 HC |
338 | static int cpu_management; |
339 | ||
72f31889 LT |
340 | static ssize_t dispatching_show(struct device *dev, |
341 | struct device_attribute *attr, | |
83a24e32 HC |
342 | char *buf) |
343 | { | |
344 | ssize_t count; | |
345 | ||
346 | mutex_lock(&smp_cpu_state_mutex); | |
347 | count = sprintf(buf, "%d\n", cpu_management); | |
348 | mutex_unlock(&smp_cpu_state_mutex); | |
349 | return count; | |
350 | } | |
351 | ||
72f31889 LT |
352 | static ssize_t dispatching_store(struct device *dev, |
353 | struct device_attribute *attr, | |
83a24e32 HC |
354 | const char *buf, |
355 | size_t count) | |
356 | { | |
357 | int val, rc; | |
358 | char delim; | |
359 | ||
360 | if (sscanf(buf, "%d %c", &val, &delim) != 1) | |
361 | return -EINVAL; | |
362 | if (val != 0 && val != 1) | |
363 | return -EINVAL; | |
364 | rc = 0; | |
365 | get_online_cpus(); | |
366 | mutex_lock(&smp_cpu_state_mutex); | |
367 | if (cpu_management == val) | |
368 | goto out; | |
369 | rc = topology_set_cpu_management(val); | |
d68bddb7 HC |
370 | if (rc) |
371 | goto out; | |
372 | cpu_management = val; | |
373 | topology_expect_change(); | |
83a24e32 HC |
374 | out: |
375 | mutex_unlock(&smp_cpu_state_mutex); | |
376 | put_online_cpus(); | |
377 | return rc ? rc : count; | |
378 | } | |
72f31889 | 379 | static DEVICE_ATTR(dispatching, 0644, dispatching_show, |
83a24e32 HC |
380 | dispatching_store); |
381 | ||
72f31889 LT |
382 | static ssize_t cpu_polarization_show(struct device *dev, |
383 | struct device_attribute *attr, char *buf) | |
83a24e32 HC |
384 | { |
385 | int cpu = dev->id; | |
386 | ssize_t count; | |
387 | ||
388 | mutex_lock(&smp_cpu_state_mutex); | |
50ab9a9a | 389 | switch (smp_cpu_get_polarization(cpu)) { |
83a24e32 HC |
390 | case POLARIZATION_HRZ: |
391 | count = sprintf(buf, "horizontal\n"); | |
392 | break; | |
393 | case POLARIZATION_VL: | |
394 | count = sprintf(buf, "vertical:low\n"); | |
395 | break; | |
396 | case POLARIZATION_VM: | |
397 | count = sprintf(buf, "vertical:medium\n"); | |
398 | break; | |
399 | case POLARIZATION_VH: | |
400 | count = sprintf(buf, "vertical:high\n"); | |
401 | break; | |
402 | default: | |
403 | count = sprintf(buf, "unknown\n"); | |
404 | break; | |
405 | } | |
406 | mutex_unlock(&smp_cpu_state_mutex); | |
407 | return count; | |
408 | } | |
72f31889 | 409 | static DEVICE_ATTR(polarization, 0444, cpu_polarization_show, NULL); |
83a24e32 HC |
410 | |
411 | static struct attribute *topology_cpu_attrs[] = { | |
72f31889 | 412 | &dev_attr_polarization.attr, |
83a24e32 HC |
413 | NULL, |
414 | }; | |
415 | ||
416 | static struct attribute_group topology_cpu_attr_group = { | |
417 | .attrs = topology_cpu_attrs, | |
418 | }; | |
419 | ||
420 | int topology_cpu_init(struct cpu *cpu) | |
421 | { | |
72f31889 | 422 | return sysfs_create_group(&cpu->dev.kobj, &topology_cpu_attr_group); |
83a24e32 HC |
423 | } |
424 | ||
3ddb1b75 | 425 | static const struct cpumask *cpu_thread_mask(int cpu) |
10ad34bc | 426 | { |
da0c636e | 427 | return &per_cpu(cpu_topology, cpu).thread_mask; |
10ad34bc MS |
428 | } |
429 | ||
430 | ||
2dfd7476 VG |
431 | const struct cpumask *cpu_coregroup_mask(int cpu) |
432 | { | |
da0c636e | 433 | return &per_cpu(cpu_topology, cpu).core_mask; |
2dfd7476 VG |
434 | } |
435 | ||
436 | static const struct cpumask *cpu_book_mask(int cpu) | |
437 | { | |
da0c636e | 438 | return &per_cpu(cpu_topology, cpu).book_mask; |
2dfd7476 VG |
439 | } |
440 | ||
d05d15da HC |
441 | static int __init early_parse_topology(char *p) |
442 | { | |
443 | if (strncmp(p, "off", 3)) | |
444 | return 0; | |
445 | topology_enabled = 0; | |
446 | return 0; | |
447 | } | |
448 | early_param("topology", early_parse_topology); | |
449 | ||
2dfd7476 | 450 | static struct sched_domain_topology_level s390_topology[] = { |
10ad34bc | 451 | { cpu_thread_mask, cpu_smt_flags, SD_INIT_NAME(SMT) }, |
2dfd7476 VG |
452 | { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) }, |
453 | { cpu_book_mask, SD_INIT_NAME(BOOK) }, | |
c0e5ddab | 454 | { cpu_cpu_mask, SD_INIT_NAME(DIE) }, |
2dfd7476 VG |
455 | { NULL, }, |
456 | }; | |
457 | ||
d05d15da HC |
458 | static void __init alloc_masks(struct sysinfo_15_1_x *info, |
459 | struct mask_info *mask, int offset) | |
460 | { | |
461 | int i, nr_masks; | |
462 | ||
463 | nr_masks = info->mag[TOPOLOGY_NR_MAG - offset]; | |
464 | for (i = 0; i < info->mnest - offset; i++) | |
465 | nr_masks *= info->mag[TOPOLOGY_NR_MAG - offset - 1 - i]; | |
466 | nr_masks = max(nr_masks, 1); | |
467 | for (i = 0; i < nr_masks; i++) { | |
468 | mask->next = kzalloc(sizeof(*mask->next), GFP_KERNEL); | |
469 | mask = mask->next; | |
470 | } | |
471 | } | |
472 | ||
473 | static int __init s390_topology_init(void) | |
474 | { | |
475 | struct sysinfo_15_1_x *info; | |
476 | int i; | |
477 | ||
478 | if (!MACHINE_HAS_TOPOLOGY) | |
479 | return 0; | |
480 | tl_info = (struct sysinfo_15_1_x *)__get_free_page(GFP_KERNEL); | |
481 | info = tl_info; | |
482 | store_topology(info); | |
483 | pr_info("The CPU configuration topology of the machine is:"); | |
484 | for (i = 0; i < TOPOLOGY_NR_MAG; i++) | |
485 | printk(KERN_CONT " %d", info->mag[i]); | |
486 | printk(KERN_CONT " / %d\n", info->mnest); | |
487 | alloc_masks(info, &socket_info, 1); | |
488 | alloc_masks(info, &book_info, 2); | |
489 | set_sched_topology(s390_topology); | |
490 | return 0; | |
491 | } | |
492 | early_initcall(s390_topology_init); | |
493 | ||
83a24e32 HC |
494 | static int __init topology_init(void) |
495 | { | |
48e9a6c1 MS |
496 | if (MACHINE_HAS_TOPOLOGY) |
497 | set_topology_timer(); | |
498 | else | |
83a24e32 | 499 | topology_update_polarization_simple(); |
72f31889 | 500 | return device_create_file(cpu_subsys.dev_root, &dev_attr_dispatching); |
83a24e32 HC |
501 | } |
502 | device_initcall(topology_init); |