2 * SMP related functions
4 * Copyright IBM Corp. 1999, 2012
5 * Author(s): Denis Joseph Barrow,
6 * Martin Schwidefsky <schwidefsky@de.ibm.com>,
7 * Heiko Carstens <heiko.carstens@de.ibm.com>,
9 * based on other smp stuff by
10 * (c) 1995 Alan Cox, CymruNET Ltd <alan@cymru.net>
11 * (c) 1998 Ingo Molnar
13 * The code outside of smp.c uses logical cpu numbers, only smp.c does
14 * the translation of logical to physical cpu ids. All new code that
15 * operates on physical cpu numbers needs to go into smp.c.
18 #define KMSG_COMPONENT "cpu"
19 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
21 #include <linux/workqueue.h>
22 #include <linux/module.h>
23 #include <linux/init.h>
25 #include <linux/err.h>
26 #include <linux/spinlock.h>
27 #include <linux/kernel_stat.h>
28 #include <linux/delay.h>
29 #include <linux/interrupt.h>
30 #include <linux/irqflags.h>
31 #include <linux/cpu.h>
32 #include <linux/slab.h>
33 #include <linux/crash_dump.h>
34 #include <linux/memblock.h>
35 #include <asm/asm-offsets.h>
37 #include <asm/switch_to.h>
38 #include <asm/facility.h>
40 #include <asm/setup.h>
42 #include <asm/tlbflush.h>
43 #include <asm/vtimer.h>
44 #include <asm/lowcore.h>
47 #include <asm/debug.h>
48 #include <asm/os_info.h>
55 ec_call_function_single
,
64 static DEFINE_PER_CPU(struct cpu
*, cpu_device
);
67 struct lowcore
*lowcore
; /* lowcore page(s) for the cpu */
68 unsigned long ec_mask
; /* bit mask for ec_xxx functions */
69 unsigned long ec_clk
; /* sigp timestamp for ec_xxx */
70 signed char state
; /* physical cpu state */
71 signed char polarization
; /* physical polarization */
72 u16 address
; /* physical cpu address */
75 static u8 boot_core_type
;
76 static struct pcpu pcpu_devices
[NR_CPUS
];
78 unsigned int smp_cpu_mt_shift
;
79 EXPORT_SYMBOL(smp_cpu_mt_shift
);
81 unsigned int smp_cpu_mtid
;
82 EXPORT_SYMBOL(smp_cpu_mtid
);
84 #ifdef CONFIG_CRASH_DUMP
85 __vector128 __initdata boot_cpu_vector_save_area
[__NUM_VXRS
];
88 static unsigned int smp_max_threads __initdata
= -1U;
90 static int __init
early_nosmt(char *s
)
95 early_param("nosmt", early_nosmt
);
97 static int __init
early_smt(char *s
)
99 get_option(&s
, &smp_max_threads
);
102 early_param("smt", early_smt
);
105 * The smp_cpu_state_mutex must be held when changing the state or polarization
106 * member of a pcpu data structure within the pcpu_devices arreay.
108 DEFINE_MUTEX(smp_cpu_state_mutex
);
111 * Signal processor helper functions.
113 static inline int __pcpu_sigp_relax(u16 addr
, u8 order
, unsigned long parm
)
118 cc
= __pcpu_sigp(addr
, order
, parm
, NULL
);
119 if (cc
!= SIGP_CC_BUSY
)
125 static int pcpu_sigp_retry(struct pcpu
*pcpu
, u8 order
, u32 parm
)
129 for (retry
= 0; ; retry
++) {
130 cc
= __pcpu_sigp(pcpu
->address
, order
, parm
, NULL
);
131 if (cc
!= SIGP_CC_BUSY
)
139 static inline int pcpu_stopped(struct pcpu
*pcpu
)
141 u32
uninitialized_var(status
);
143 if (__pcpu_sigp(pcpu
->address
, SIGP_SENSE
,
144 0, &status
) != SIGP_CC_STATUS_STORED
)
146 return !!(status
& (SIGP_STATUS_CHECK_STOP
|SIGP_STATUS_STOPPED
));
149 static inline int pcpu_running(struct pcpu
*pcpu
)
151 if (__pcpu_sigp(pcpu
->address
, SIGP_SENSE_RUNNING
,
152 0, NULL
) != SIGP_CC_STATUS_STORED
)
154 /* Status stored condition code is equivalent to cpu not running. */
159 * Find struct pcpu by cpu address.
161 static struct pcpu
*pcpu_find_address(const struct cpumask
*mask
, u16 address
)
165 for_each_cpu(cpu
, mask
)
166 if (pcpu_devices
[cpu
].address
== address
)
167 return pcpu_devices
+ cpu
;
171 static void pcpu_ec_call(struct pcpu
*pcpu
, int ec_bit
)
175 if (test_and_set_bit(ec_bit
, &pcpu
->ec_mask
))
177 order
= pcpu_running(pcpu
) ? SIGP_EXTERNAL_CALL
: SIGP_EMERGENCY_SIGNAL
;
178 pcpu
->ec_clk
= get_tod_clock_fast();
179 pcpu_sigp_retry(pcpu
, order
, 0);
182 #define ASYNC_FRAME_OFFSET (ASYNC_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE)
183 #define PANIC_FRAME_OFFSET (PAGE_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE)
185 static int pcpu_alloc_lowcore(struct pcpu
*pcpu
, int cpu
)
187 unsigned long async_stack
, panic_stack
;
190 if (pcpu
!= &pcpu_devices
[0]) {
191 pcpu
->lowcore
= (struct lowcore
*)
192 __get_free_pages(GFP_KERNEL
| GFP_DMA
, LC_ORDER
);
193 async_stack
= __get_free_pages(GFP_KERNEL
, ASYNC_ORDER
);
194 panic_stack
= __get_free_page(GFP_KERNEL
);
195 if (!pcpu
->lowcore
|| !panic_stack
|| !async_stack
)
198 async_stack
= pcpu
->lowcore
->async_stack
- ASYNC_FRAME_OFFSET
;
199 panic_stack
= pcpu
->lowcore
->panic_stack
- PANIC_FRAME_OFFSET
;
202 memcpy(lc
, &S390_lowcore
, 512);
203 memset((char *) lc
+ 512, 0, sizeof(*lc
) - 512);
204 lc
->async_stack
= async_stack
+ ASYNC_FRAME_OFFSET
;
205 lc
->panic_stack
= panic_stack
+ PANIC_FRAME_OFFSET
;
207 lc
->spinlock_lockval
= arch_spin_lockval(cpu
);
209 lc
->vector_save_area_addr
=
210 (unsigned long) &lc
->vector_save_area
;
211 if (vdso_alloc_per_cpu(lc
))
213 lowcore_ptr
[cpu
] = lc
;
214 pcpu_sigp_retry(pcpu
, SIGP_SET_PREFIX
, (u32
)(unsigned long) lc
);
217 if (pcpu
!= &pcpu_devices
[0]) {
218 free_page(panic_stack
);
219 free_pages(async_stack
, ASYNC_ORDER
);
220 free_pages((unsigned long) pcpu
->lowcore
, LC_ORDER
);
225 #ifdef CONFIG_HOTPLUG_CPU
227 static void pcpu_free_lowcore(struct pcpu
*pcpu
)
229 pcpu_sigp_retry(pcpu
, SIGP_SET_PREFIX
, 0);
230 lowcore_ptr
[pcpu
- pcpu_devices
] = NULL
;
231 vdso_free_per_cpu(pcpu
->lowcore
);
232 if (pcpu
== &pcpu_devices
[0])
234 free_page(pcpu
->lowcore
->panic_stack
-PANIC_FRAME_OFFSET
);
235 free_pages(pcpu
->lowcore
->async_stack
-ASYNC_FRAME_OFFSET
, ASYNC_ORDER
);
236 free_pages((unsigned long) pcpu
->lowcore
, LC_ORDER
);
239 #endif /* CONFIG_HOTPLUG_CPU */
241 static void pcpu_prepare_secondary(struct pcpu
*pcpu
, int cpu
)
243 struct lowcore
*lc
= pcpu
->lowcore
;
245 if (MACHINE_HAS_TLB_LC
)
246 cpumask_set_cpu(cpu
, &init_mm
.context
.cpu_attach_mask
);
247 cpumask_set_cpu(cpu
, mm_cpumask(&init_mm
));
248 atomic_inc(&init_mm
.context
.attach_count
);
250 lc
->spinlock_lockval
= arch_spin_lockval(cpu
);
251 lc
->percpu_offset
= __per_cpu_offset
[cpu
];
252 lc
->kernel_asce
= S390_lowcore
.kernel_asce
;
253 lc
->machine_flags
= S390_lowcore
.machine_flags
;
254 lc
->user_timer
= lc
->system_timer
= lc
->steal_timer
= 0;
255 __ctl_store(lc
->cregs_save_area
, 0, 15);
256 save_access_regs((unsigned int *) lc
->access_regs_save_area
);
257 memcpy(lc
->stfle_fac_list
, S390_lowcore
.stfle_fac_list
,
261 static void pcpu_attach_task(struct pcpu
*pcpu
, struct task_struct
*tsk
)
263 struct lowcore
*lc
= pcpu
->lowcore
;
264 struct thread_info
*ti
= task_thread_info(tsk
);
266 lc
->kernel_stack
= (unsigned long) task_stack_page(tsk
)
267 + THREAD_SIZE
- STACK_FRAME_OVERHEAD
- sizeof(struct pt_regs
);
268 lc
->thread_info
= (unsigned long) task_thread_info(tsk
);
269 lc
->current_task
= (unsigned long) tsk
;
271 lc
->current_pid
= tsk
->pid
;
272 lc
->user_timer
= ti
->user_timer
;
273 lc
->system_timer
= ti
->system_timer
;
277 static void pcpu_start_fn(struct pcpu
*pcpu
, void (*func
)(void *), void *data
)
279 struct lowcore
*lc
= pcpu
->lowcore
;
281 lc
->restart_stack
= lc
->kernel_stack
;
282 lc
->restart_fn
= (unsigned long) func
;
283 lc
->restart_data
= (unsigned long) data
;
284 lc
->restart_source
= -1UL;
285 pcpu_sigp_retry(pcpu
, SIGP_RESTART
, 0);
289 * Call function via PSW restart on pcpu and stop the current cpu.
291 static void pcpu_delegate(struct pcpu
*pcpu
, void (*func
)(void *),
292 void *data
, unsigned long stack
)
294 struct lowcore
*lc
= lowcore_ptr
[pcpu
- pcpu_devices
];
295 unsigned long source_cpu
= stap();
297 __load_psw_mask(PSW_KERNEL_BITS
);
298 if (pcpu
->address
== source_cpu
)
299 func(data
); /* should not return */
300 /* Stop target cpu (if func returns this stops the current cpu). */
301 pcpu_sigp_retry(pcpu
, SIGP_STOP
, 0);
302 /* Restart func on the target cpu and stop the current cpu. */
303 mem_assign_absolute(lc
->restart_stack
, stack
);
304 mem_assign_absolute(lc
->restart_fn
, (unsigned long) func
);
305 mem_assign_absolute(lc
->restart_data
, (unsigned long) data
);
306 mem_assign_absolute(lc
->restart_source
, source_cpu
);
308 "0: sigp 0,%0,%2 # sigp restart to target cpu\n"
309 " brc 2,0b # busy, try again\n"
310 "1: sigp 0,%1,%3 # sigp stop to current cpu\n"
311 " brc 2,1b # busy, try again\n"
312 : : "d" (pcpu
->address
), "d" (source_cpu
),
313 "K" (SIGP_RESTART
), "K" (SIGP_STOP
)
319 * Enable additional logical cpus for multi-threading.
321 static int pcpu_set_smt(unsigned int mtid
)
323 register unsigned long reg1
asm ("1") = (unsigned long) mtid
;
326 if (smp_cpu_mtid
== mtid
)
329 " sigp %1,0,%2 # sigp set multi-threading\n"
332 : "=d" (cc
) : "d" (reg1
), "K" (SIGP_SET_MULTI_THREADING
)
336 smp_cpu_mt_shift
= 0;
337 while (smp_cpu_mtid
>= (1U << smp_cpu_mt_shift
))
339 pcpu_devices
[0].address
= stap();
345 * Call function on an online CPU.
347 void smp_call_online_cpu(void (*func
)(void *), void *data
)
351 /* Use the current cpu if it is online. */
352 pcpu
= pcpu_find_address(cpu_online_mask
, stap());
354 /* Use the first online cpu. */
355 pcpu
= pcpu_devices
+ cpumask_first(cpu_online_mask
);
356 pcpu_delegate(pcpu
, func
, data
, (unsigned long) restart_stack
);
360 * Call function on the ipl CPU.
362 void smp_call_ipl_cpu(void (*func
)(void *), void *data
)
364 pcpu_delegate(&pcpu_devices
[0], func
, data
,
365 pcpu_devices
->lowcore
->panic_stack
-
366 PANIC_FRAME_OFFSET
+ PAGE_SIZE
);
369 int smp_find_processor_id(u16 address
)
373 for_each_present_cpu(cpu
)
374 if (pcpu_devices
[cpu
].address
== address
)
379 int smp_vcpu_scheduled(int cpu
)
381 return pcpu_running(pcpu_devices
+ cpu
);
384 void smp_yield_cpu(int cpu
)
386 if (MACHINE_HAS_DIAG9C
) {
387 diag_stat_inc_norecursion(DIAG_STAT_X09C
);
388 asm volatile("diag %0,0,0x9c"
389 : : "d" (pcpu_devices
[cpu
].address
));
390 } else if (MACHINE_HAS_DIAG44
) {
391 diag_stat_inc_norecursion(DIAG_STAT_X044
);
392 asm volatile("diag 0,0,0x44");
397 * Send cpus emergency shutdown signal. This gives the cpus the
398 * opportunity to complete outstanding interrupts.
400 static void smp_emergency_stop(cpumask_t
*cpumask
)
405 end
= get_tod_clock() + (1000000UL << 12);
406 for_each_cpu(cpu
, cpumask
) {
407 struct pcpu
*pcpu
= pcpu_devices
+ cpu
;
408 set_bit(ec_stop_cpu
, &pcpu
->ec_mask
);
409 while (__pcpu_sigp(pcpu
->address
, SIGP_EMERGENCY_SIGNAL
,
410 0, NULL
) == SIGP_CC_BUSY
&&
411 get_tod_clock() < end
)
414 while (get_tod_clock() < end
) {
415 for_each_cpu(cpu
, cpumask
)
416 if (pcpu_stopped(pcpu_devices
+ cpu
))
417 cpumask_clear_cpu(cpu
, cpumask
);
418 if (cpumask_empty(cpumask
))
425 * Stop all cpus but the current one.
427 void smp_send_stop(void)
432 /* Disable all interrupts/machine checks */
433 __load_psw_mask(PSW_KERNEL_BITS
| PSW_MASK_DAT
);
434 trace_hardirqs_off();
436 debug_set_critical();
437 cpumask_copy(&cpumask
, cpu_online_mask
);
438 cpumask_clear_cpu(smp_processor_id(), &cpumask
);
440 if (oops_in_progress
)
441 smp_emergency_stop(&cpumask
);
443 /* stop all processors */
444 for_each_cpu(cpu
, &cpumask
) {
445 struct pcpu
*pcpu
= pcpu_devices
+ cpu
;
446 pcpu_sigp_retry(pcpu
, SIGP_STOP
, 0);
447 while (!pcpu_stopped(pcpu
))
453 * This is the main routine where commands issued by other
456 static void smp_handle_ext_call(void)
460 /* handle bit signal external calls */
461 bits
= xchg(&pcpu_devices
[smp_processor_id()].ec_mask
, 0);
462 if (test_bit(ec_stop_cpu
, &bits
))
464 if (test_bit(ec_schedule
, &bits
))
466 if (test_bit(ec_call_function_single
, &bits
))
467 generic_smp_call_function_single_interrupt();
470 static void do_ext_call_interrupt(struct ext_code ext_code
,
471 unsigned int param32
, unsigned long param64
)
473 inc_irq_stat(ext_code
.code
== 0x1202 ? IRQEXT_EXC
: IRQEXT_EMS
);
474 smp_handle_ext_call();
477 void arch_send_call_function_ipi_mask(const struct cpumask
*mask
)
481 for_each_cpu(cpu
, mask
)
482 pcpu_ec_call(pcpu_devices
+ cpu
, ec_call_function_single
);
485 void arch_send_call_function_single_ipi(int cpu
)
487 pcpu_ec_call(pcpu_devices
+ cpu
, ec_call_function_single
);
491 * this function sends a 'reschedule' IPI to another CPU.
492 * it goes straight through and wastes no time serializing
493 * anything. Worst case is that we lose a reschedule ...
495 void smp_send_reschedule(int cpu
)
497 pcpu_ec_call(pcpu_devices
+ cpu
, ec_schedule
);
501 * parameter area for the set/clear control bit callbacks
503 struct ec_creg_mask_parms
{
505 unsigned long andval
;
510 * callback for setting/clearing control bits
512 static void smp_ctl_bit_callback(void *info
)
514 struct ec_creg_mask_parms
*pp
= info
;
515 unsigned long cregs
[16];
517 __ctl_store(cregs
, 0, 15);
518 cregs
[pp
->cr
] = (cregs
[pp
->cr
] & pp
->andval
) | pp
->orval
;
519 __ctl_load(cregs
, 0, 15);
523 * Set a bit in a control register of all cpus
525 void smp_ctl_set_bit(int cr
, int bit
)
527 struct ec_creg_mask_parms parms
= { 1UL << bit
, -1UL, cr
};
529 on_each_cpu(smp_ctl_bit_callback
, &parms
, 1);
531 EXPORT_SYMBOL(smp_ctl_set_bit
);
534 * Clear a bit in a control register of all cpus
536 void smp_ctl_clear_bit(int cr
, int bit
)
538 struct ec_creg_mask_parms parms
= { 0, ~(1UL << bit
), cr
};
540 on_each_cpu(smp_ctl_bit_callback
, &parms
, 1);
542 EXPORT_SYMBOL(smp_ctl_clear_bit
);
544 #ifdef CONFIG_CRASH_DUMP
546 int smp_store_status(int cpu
)
548 struct pcpu
*pcpu
= pcpu_devices
+ cpu
;
551 pa
= __pa(&pcpu
->lowcore
->floating_pt_save_area
);
552 if (__pcpu_sigp_relax(pcpu
->address
, SIGP_STORE_STATUS_AT_ADDRESS
,
553 pa
) != SIGP_CC_ORDER_CODE_ACCEPTED
)
557 pa
= __pa(pcpu
->lowcore
->vector_save_area_addr
);
558 if (__pcpu_sigp_relax(pcpu
->address
, SIGP_STORE_ADDITIONAL_STATUS
,
559 pa
) != SIGP_CC_ORDER_CODE_ACCEPTED
)
565 * Collect CPU state of the previous, crashed system.
566 * There are four cases:
567 * 1) standard zfcp dump
568 * condition: OLDMEM_BASE == NULL && ipl_info.type == IPL_TYPE_FCP_DUMP
569 * The state for all CPUs except the boot CPU needs to be collected
570 * with sigp stop-and-store-status. The boot CPU state is located in
571 * the absolute lowcore of the memory stored in the HSA. The zcore code
572 * will copy the boot CPU state from the HSA.
573 * 2) stand-alone kdump for SCSI (zfcp dump with swapped memory)
574 * condition: OLDMEM_BASE != NULL && ipl_info.type == IPL_TYPE_FCP_DUMP
575 * The state for all CPUs except the boot CPU needs to be collected
576 * with sigp stop-and-store-status. The firmware or the boot-loader
577 * stored the registers of the boot CPU in the absolute lowcore in the
578 * memory of the old system.
579 * 3) kdump and the old kernel did not store the CPU state,
580 * or stand-alone kdump for DASD
581 * condition: OLDMEM_BASE != NULL && !is_kdump_kernel()
582 * The state for all CPUs except the boot CPU needs to be collected
583 * with sigp stop-and-store-status. The kexec code or the boot-loader
584 * stored the registers of the boot CPU in the memory of the old system.
585 * 4) kdump and the old kernel stored the CPU state
586 * condition: OLDMEM_BASE != NULL && is_kdump_kernel()
587 * This case does not exist for s390 anymore, setup_arch explicitly
588 * deactivates the elfcorehdr= kernel parameter
590 static __init
void smp_save_cpu_vxrs(struct save_area
*sa
, u16 addr
,
591 bool is_boot_cpu
, unsigned long page
)
593 __vector128
*vxrs
= (__vector128
*) page
;
596 vxrs
= boot_cpu_vector_save_area
;
598 __pcpu_sigp_relax(addr
, SIGP_STORE_ADDITIONAL_STATUS
, page
);
599 save_area_add_vxrs(sa
, vxrs
);
602 static __init
void smp_save_cpu_regs(struct save_area
*sa
, u16 addr
,
603 bool is_boot_cpu
, unsigned long page
)
605 void *regs
= (void *) page
;
608 copy_oldmem_kernel(regs
, (void *) __LC_FPREGS_SAVE_AREA
, 512);
610 __pcpu_sigp_relax(addr
, SIGP_STORE_STATUS_AT_ADDRESS
, page
);
611 save_area_add_regs(sa
, regs
);
614 void __init
smp_save_dump_cpus(void)
616 int addr
, boot_cpu_addr
, max_cpu_addr
;
617 struct save_area
*sa
;
621 if (!(OLDMEM_BASE
|| ipl_info
.type
== IPL_TYPE_FCP_DUMP
))
622 /* No previous system present, normal boot. */
624 /* Allocate a page as dumping area for the store status sigps */
625 page
= memblock_alloc_base(PAGE_SIZE
, PAGE_SIZE
, 1UL << 31);
627 panic("could not allocate memory for save area\n");
628 /* Set multi-threading state to the previous system. */
629 pcpu_set_smt(sclp
.mtid_prev
);
630 boot_cpu_addr
= stap();
631 max_cpu_addr
= SCLP_MAX_CORES
<< sclp
.mtid_prev
;
632 for (addr
= 0; addr
<= max_cpu_addr
; addr
++) {
633 if (__pcpu_sigp_relax(addr
, SIGP_SENSE
, 0) ==
634 SIGP_CC_NOT_OPERATIONAL
)
636 is_boot_cpu
= (addr
== boot_cpu_addr
);
637 /* Allocate save area */
638 sa
= save_area_alloc(is_boot_cpu
);
640 panic("could not allocate memory for save area\n");
642 /* Get the vector registers */
643 smp_save_cpu_vxrs(sa
, addr
, is_boot_cpu
, page
);
645 * For a zfcp dump OLDMEM_BASE == NULL and the registers
646 * of the boot CPU are stored in the HSA. To retrieve
647 * these registers an SCLP request is required which is
648 * done by drivers/s390/char/zcore.c:init_cpu_info()
650 if (!is_boot_cpu
|| OLDMEM_BASE
)
651 /* Get the CPU registers */
652 smp_save_cpu_regs(sa
, addr
, is_boot_cpu
, page
);
654 memblock_free(page
, PAGE_SIZE
);
658 #endif /* CONFIG_CRASH_DUMP */
660 void smp_cpu_set_polarization(int cpu
, int val
)
662 pcpu_devices
[cpu
].polarization
= val
;
665 int smp_cpu_get_polarization(int cpu
)
667 return pcpu_devices
[cpu
].polarization
;
670 static struct sclp_core_info
*smp_get_core_info(void)
672 static int use_sigp_detection
;
673 struct sclp_core_info
*info
;
676 info
= kzalloc(sizeof(*info
), GFP_KERNEL
);
677 if (info
&& (use_sigp_detection
|| sclp_get_core_info(info
))) {
678 use_sigp_detection
= 1;
680 address
< (SCLP_MAX_CORES
<< smp_cpu_mt_shift
);
681 address
+= (1U << smp_cpu_mt_shift
)) {
682 if (__pcpu_sigp_relax(address
, SIGP_SENSE
, 0) ==
683 SIGP_CC_NOT_OPERATIONAL
)
685 info
->core
[info
->configured
].core_id
=
686 address
>> smp_cpu_mt_shift
;
689 info
->combined
= info
->configured
;
694 static int smp_add_present_cpu(int cpu
);
696 static int __smp_rescan_cpus(struct sclp_core_info
*info
, int sysfs_add
)
704 cpumask_xor(&avail
, cpu_possible_mask
, cpu_present_mask
);
705 cpu
= cpumask_first(&avail
);
706 for (i
= 0; (i
< info
->combined
) && (cpu
< nr_cpu_ids
); i
++) {
707 if (sclp
.has_core_type
&& info
->core
[i
].type
!= boot_core_type
)
709 address
= info
->core
[i
].core_id
<< smp_cpu_mt_shift
;
710 for (j
= 0; j
<= smp_cpu_mtid
; j
++) {
711 if (pcpu_find_address(cpu_present_mask
, address
+ j
))
713 pcpu
= pcpu_devices
+ cpu
;
714 pcpu
->address
= address
+ j
;
716 (cpu
>= info
->configured
*(smp_cpu_mtid
+ 1)) ?
717 CPU_STATE_STANDBY
: CPU_STATE_CONFIGURED
;
718 smp_cpu_set_polarization(cpu
, POLARIZATION_UNKNOWN
);
719 set_cpu_present(cpu
, true);
720 if (sysfs_add
&& smp_add_present_cpu(cpu
) != 0)
721 set_cpu_present(cpu
, false);
724 cpu
= cpumask_next(cpu
, &avail
);
725 if (cpu
>= nr_cpu_ids
)
732 static void __init
smp_detect_cpus(void)
734 unsigned int cpu
, mtid
, c_cpus
, s_cpus
;
735 struct sclp_core_info
*info
;
738 /* Get CPU information */
739 info
= smp_get_core_info();
741 panic("smp_detect_cpus failed to allocate memory\n");
743 /* Find boot CPU type */
744 if (sclp
.has_core_type
) {
746 for (cpu
= 0; cpu
< info
->combined
; cpu
++)
747 if (info
->core
[cpu
].core_id
== address
) {
748 /* The boot cpu dictates the cpu type. */
749 boot_core_type
= info
->core
[cpu
].type
;
752 if (cpu
>= info
->combined
)
753 panic("Could not find boot CPU type");
756 /* Set multi-threading state for the current system */
757 mtid
= boot_core_type
? sclp
.mtid
: sclp
.mtid_cp
;
758 mtid
= (mtid
< smp_max_threads
) ? mtid
: smp_max_threads
- 1;
761 /* Print number of CPUs */
763 for (cpu
= 0; cpu
< info
->combined
; cpu
++) {
764 if (sclp
.has_core_type
&&
765 info
->core
[cpu
].type
!= boot_core_type
)
767 if (cpu
< info
->configured
)
768 c_cpus
+= smp_cpu_mtid
+ 1;
770 s_cpus
+= smp_cpu_mtid
+ 1;
772 pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus
, s_cpus
);
774 /* Add CPUs present at boot */
776 __smp_rescan_cpus(info
, 0);
782 * Activate a secondary processor.
784 static void smp_start_secondary(void *cpuvoid
)
786 S390_lowcore
.last_update_clock
= get_tod_clock();
787 S390_lowcore
.restart_stack
= (unsigned long) restart_stack
;
788 S390_lowcore
.restart_fn
= (unsigned long) do_restart
;
789 S390_lowcore
.restart_data
= 0;
790 S390_lowcore
.restart_source
= -1UL;
791 restore_access_regs(S390_lowcore
.access_regs_save_area
);
792 __ctl_load(S390_lowcore
.cregs_save_area
, 0, 15);
793 __load_psw_mask(PSW_KERNEL_BITS
| PSW_MASK_DAT
);
799 notify_cpu_starting(smp_processor_id());
800 set_cpu_online(smp_processor_id(), true);
801 inc_irq_stat(CPU_RST
);
803 cpu_startup_entry(CPUHP_ONLINE
);
806 /* Upping and downing of CPUs */
807 int __cpu_up(unsigned int cpu
, struct task_struct
*tidle
)
812 pcpu
= pcpu_devices
+ cpu
;
813 if (pcpu
->state
!= CPU_STATE_CONFIGURED
)
815 base
= cpu
- (cpu
% (smp_cpu_mtid
+ 1));
816 for (i
= 0; i
<= smp_cpu_mtid
; i
++) {
817 if (base
+ i
< nr_cpu_ids
)
818 if (cpu_online(base
+ i
))
822 * If this is the first CPU of the core to get online
823 * do an initial CPU reset.
825 if (i
> smp_cpu_mtid
&&
826 pcpu_sigp_retry(pcpu_devices
+ base
, SIGP_INITIAL_CPU_RESET
, 0) !=
827 SIGP_CC_ORDER_CODE_ACCEPTED
)
830 rc
= pcpu_alloc_lowcore(pcpu
, cpu
);
833 pcpu_prepare_secondary(pcpu
, cpu
);
834 pcpu_attach_task(pcpu
, tidle
);
835 pcpu_start_fn(pcpu
, smp_start_secondary
, NULL
);
836 /* Wait until cpu puts itself in the online & active maps */
837 while (!cpu_online(cpu
) || !cpu_active(cpu
))
842 static unsigned int setup_possible_cpus __initdata
;
844 static int __init
_setup_possible_cpus(char *s
)
846 get_option(&s
, &setup_possible_cpus
);
849 early_param("possible_cpus", _setup_possible_cpus
);
851 #ifdef CONFIG_HOTPLUG_CPU
853 int __cpu_disable(void)
855 unsigned long cregs
[16];
857 /* Handle possible pending IPIs */
858 smp_handle_ext_call();
859 set_cpu_online(smp_processor_id(), false);
860 /* Disable pseudo page faults on this cpu. */
862 /* Disable interrupt sources via control register. */
863 __ctl_store(cregs
, 0, 15);
864 cregs
[0] &= ~0x0000ee70UL
; /* disable all external interrupts */
865 cregs
[6] &= ~0xff000000UL
; /* disable all I/O interrupts */
866 cregs
[14] &= ~0x1f000000UL
; /* disable most machine checks */
867 __ctl_load(cregs
, 0, 15);
868 clear_cpu_flag(CIF_NOHZ_DELAY
);
872 void __cpu_die(unsigned int cpu
)
876 /* Wait until target cpu is down */
877 pcpu
= pcpu_devices
+ cpu
;
878 while (!pcpu_stopped(pcpu
))
880 pcpu_free_lowcore(pcpu
);
881 atomic_dec(&init_mm
.context
.attach_count
);
882 cpumask_clear_cpu(cpu
, mm_cpumask(&init_mm
));
883 if (MACHINE_HAS_TLB_LC
)
884 cpumask_clear_cpu(cpu
, &init_mm
.context
.cpu_attach_mask
);
887 void __noreturn
cpu_die(void)
890 pcpu_sigp_retry(pcpu_devices
+ smp_processor_id(), SIGP_STOP
, 0);
894 #endif /* CONFIG_HOTPLUG_CPU */
896 void __init
smp_fill_possible_mask(void)
898 unsigned int possible
, sclp_max
, cpu
;
900 sclp_max
= max(sclp
.mtid
, sclp
.mtid_cp
) + 1;
901 sclp_max
= min(smp_max_threads
, sclp_max
);
902 sclp_max
= sclp
.max_cores
* sclp_max
?: nr_cpu_ids
;
903 possible
= setup_possible_cpus
?: nr_cpu_ids
;
904 possible
= min(possible
, sclp_max
);
905 for (cpu
= 0; cpu
< possible
&& cpu
< nr_cpu_ids
; cpu
++)
906 set_cpu_possible(cpu
, true);
909 void __init
smp_prepare_cpus(unsigned int max_cpus
)
911 /* request the 0x1201 emergency signal external interrupt */
912 if (register_external_irq(EXT_IRQ_EMERGENCY_SIG
, do_ext_call_interrupt
))
913 panic("Couldn't request external interrupt 0x1201");
914 /* request the 0x1202 external call external interrupt */
915 if (register_external_irq(EXT_IRQ_EXTERNAL_CALL
, do_ext_call_interrupt
))
916 panic("Couldn't request external interrupt 0x1202");
920 void __init
smp_prepare_boot_cpu(void)
922 struct pcpu
*pcpu
= pcpu_devices
;
924 pcpu
->state
= CPU_STATE_CONFIGURED
;
925 pcpu
->address
= stap();
926 pcpu
->lowcore
= (struct lowcore
*)(unsigned long) store_prefix();
927 S390_lowcore
.percpu_offset
= __per_cpu_offset
[0];
928 smp_cpu_set_polarization(0, POLARIZATION_UNKNOWN
);
929 set_cpu_present(0, true);
930 set_cpu_online(0, true);
933 void __init
smp_cpus_done(unsigned int max_cpus
)
937 void __init
smp_setup_processor_id(void)
939 S390_lowcore
.cpu_nr
= 0;
940 S390_lowcore
.spinlock_lockval
= arch_spin_lockval(0);
944 * the frequency of the profiling timer can be changed
945 * by writing a multiplier value into /proc/profile.
947 * usually you want to run this on all CPUs ;)
949 int setup_profiling_timer(unsigned int multiplier
)
954 #ifdef CONFIG_HOTPLUG_CPU
955 static ssize_t
cpu_configure_show(struct device
*dev
,
956 struct device_attribute
*attr
, char *buf
)
960 mutex_lock(&smp_cpu_state_mutex
);
961 count
= sprintf(buf
, "%d\n", pcpu_devices
[dev
->id
].state
);
962 mutex_unlock(&smp_cpu_state_mutex
);
966 static ssize_t
cpu_configure_store(struct device
*dev
,
967 struct device_attribute
*attr
,
968 const char *buf
, size_t count
)
974 if (sscanf(buf
, "%d %c", &val
, &delim
) != 1)
976 if (val
!= 0 && val
!= 1)
979 mutex_lock(&smp_cpu_state_mutex
);
981 /* disallow configuration changes of online cpus and cpu 0 */
983 cpu
-= cpu
% (smp_cpu_mtid
+ 1);
986 for (i
= 0; i
<= smp_cpu_mtid
; i
++)
987 if (cpu_online(cpu
+ i
))
989 pcpu
= pcpu_devices
+ cpu
;
993 if (pcpu
->state
!= CPU_STATE_CONFIGURED
)
995 rc
= sclp_core_deconfigure(pcpu
->address
>> smp_cpu_mt_shift
);
998 for (i
= 0; i
<= smp_cpu_mtid
; i
++) {
999 if (cpu
+ i
>= nr_cpu_ids
|| !cpu_present(cpu
+ i
))
1001 pcpu
[i
].state
= CPU_STATE_STANDBY
;
1002 smp_cpu_set_polarization(cpu
+ i
,
1003 POLARIZATION_UNKNOWN
);
1005 topology_expect_change();
1008 if (pcpu
->state
!= CPU_STATE_STANDBY
)
1010 rc
= sclp_core_configure(pcpu
->address
>> smp_cpu_mt_shift
);
1013 for (i
= 0; i
<= smp_cpu_mtid
; i
++) {
1014 if (cpu
+ i
>= nr_cpu_ids
|| !cpu_present(cpu
+ i
))
1016 pcpu
[i
].state
= CPU_STATE_CONFIGURED
;
1017 smp_cpu_set_polarization(cpu
+ i
,
1018 POLARIZATION_UNKNOWN
);
1020 topology_expect_change();
1026 mutex_unlock(&smp_cpu_state_mutex
);
1028 return rc
? rc
: count
;
1030 static DEVICE_ATTR(configure
, 0644, cpu_configure_show
, cpu_configure_store
);
1031 #endif /* CONFIG_HOTPLUG_CPU */
1033 static ssize_t
show_cpu_address(struct device
*dev
,
1034 struct device_attribute
*attr
, char *buf
)
1036 return sprintf(buf
, "%d\n", pcpu_devices
[dev
->id
].address
);
1038 static DEVICE_ATTR(address
, 0444, show_cpu_address
, NULL
);
1040 static struct attribute
*cpu_common_attrs
[] = {
1041 #ifdef CONFIG_HOTPLUG_CPU
1042 &dev_attr_configure
.attr
,
1044 &dev_attr_address
.attr
,
1048 static struct attribute_group cpu_common_attr_group
= {
1049 .attrs
= cpu_common_attrs
,
1052 static struct attribute
*cpu_online_attrs
[] = {
1053 &dev_attr_idle_count
.attr
,
1054 &dev_attr_idle_time_us
.attr
,
1058 static struct attribute_group cpu_online_attr_group
= {
1059 .attrs
= cpu_online_attrs
,
1062 static int smp_cpu_notify(struct notifier_block
*self
, unsigned long action
,
1065 unsigned int cpu
= (unsigned int)(long)hcpu
;
1066 struct device
*s
= &per_cpu(cpu_device
, cpu
)->dev
;
1069 switch (action
& ~CPU_TASKS_FROZEN
) {
1071 err
= sysfs_create_group(&s
->kobj
, &cpu_online_attr_group
);
1074 sysfs_remove_group(&s
->kobj
, &cpu_online_attr_group
);
1077 return notifier_from_errno(err
);
1080 static int smp_add_present_cpu(int cpu
)
1086 c
= kzalloc(sizeof(*c
), GFP_KERNEL
);
1089 per_cpu(cpu_device
, cpu
) = c
;
1091 c
->hotpluggable
= 1;
1092 rc
= register_cpu(c
, cpu
);
1095 rc
= sysfs_create_group(&s
->kobj
, &cpu_common_attr_group
);
1098 if (cpu_online(cpu
)) {
1099 rc
= sysfs_create_group(&s
->kobj
, &cpu_online_attr_group
);
1103 rc
= topology_cpu_init(c
);
1109 if (cpu_online(cpu
))
1110 sysfs_remove_group(&s
->kobj
, &cpu_online_attr_group
);
1112 sysfs_remove_group(&s
->kobj
, &cpu_common_attr_group
);
1114 #ifdef CONFIG_HOTPLUG_CPU
1121 #ifdef CONFIG_HOTPLUG_CPU
1123 int __ref
smp_rescan_cpus(void)
1125 struct sclp_core_info
*info
;
1128 info
= smp_get_core_info();
1132 mutex_lock(&smp_cpu_state_mutex
);
1133 nr
= __smp_rescan_cpus(info
, 1);
1134 mutex_unlock(&smp_cpu_state_mutex
);
1138 topology_schedule_update();
1142 static ssize_t __ref
rescan_store(struct device
*dev
,
1143 struct device_attribute
*attr
,
1149 rc
= smp_rescan_cpus();
1150 return rc
? rc
: count
;
1152 static DEVICE_ATTR(rescan
, 0200, NULL
, rescan_store
);
1153 #endif /* CONFIG_HOTPLUG_CPU */
1155 static int __init
s390_smp_init(void)
1159 #ifdef CONFIG_HOTPLUG_CPU
1160 rc
= device_create_file(cpu_subsys
.dev_root
, &dev_attr_rescan
);
1164 cpu_notifier_register_begin();
1165 for_each_present_cpu(cpu
) {
1166 rc
= smp_add_present_cpu(cpu
);
1171 __hotcpu_notifier(smp_cpu_notify
, 0);
1174 cpu_notifier_register_done();
1177 subsys_initcall(s390_smp_init
);