]>
Commit | Line | Data |
---|---|---|
99106986 GR |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
3 | #include <linux/module.h> | |
4 | #include <linux/init.h> | |
5 | #include <linux/kernel.h> | |
6 | #include <linux/mm.h> | |
7 | #include <linux/sched.h> | |
8 | #include <linux/kernel_stat.h> | |
9 | #include <linux/notifier.h> | |
10 | #include <linux/cpu.h> | |
11 | #include <linux/percpu.h> | |
12 | #include <linux/delay.h> | |
13 | #include <linux/err.h> | |
14 | #include <linux/irq.h> | |
2c81b076 | 15 | #include <linux/irq_work.h> |
99106986 GR |
16 | #include <linux/irqdomain.h> |
17 | #include <linux/of.h> | |
e6169c4b | 18 | #include <linux/seq_file.h> |
99106986 GR |
19 | #include <linux/sched/task_stack.h> |
20 | #include <linux/sched/mm.h> | |
859e5f45 | 21 | #include <linux/sched/hotplug.h> |
99106986 GR |
22 | #include <asm/irq.h> |
23 | #include <asm/traps.h> | |
24 | #include <asm/sections.h> | |
25 | #include <asm/mmu_context.h> | |
12879bda GR |
26 | #ifdef CONFIG_CPU_HAS_FPU |
27 | #include <abi/fpu.h> | |
28 | #endif | |
99106986 | 29 | |
99106986 GR |
30 | enum ipi_message_type { |
31 | IPI_EMPTY, | |
32 | IPI_RESCHEDULE, | |
33 | IPI_CALL_FUNC, | |
2c81b076 | 34 | IPI_IRQ_WORK, |
99106986 GR |
35 | IPI_MAX |
36 | }; | |
37 | ||
e6169c4b GR |
38 | struct ipi_data_struct { |
39 | unsigned long bits ____cacheline_aligned; | |
40 | unsigned long stats[IPI_MAX] ____cacheline_aligned; | |
41 | }; | |
42 | static DEFINE_PER_CPU(struct ipi_data_struct, ipi_data); | |
43 | ||
99106986 GR |
44 | static irqreturn_t handle_ipi(int irq, void *dev) |
45 | { | |
e6169c4b GR |
46 | unsigned long *stats = this_cpu_ptr(&ipi_data)->stats; |
47 | ||
99106986 GR |
48 | while (true) { |
49 | unsigned long ops; | |
50 | ||
51 | ops = xchg(&this_cpu_ptr(&ipi_data)->bits, 0); | |
52 | if (ops == 0) | |
53 | return IRQ_HANDLED; | |
54 | ||
e6169c4b GR |
55 | if (ops & (1 << IPI_RESCHEDULE)) { |
56 | stats[IPI_RESCHEDULE]++; | |
99106986 | 57 | scheduler_ipi(); |
e6169c4b | 58 | } |
99106986 | 59 | |
e6169c4b GR |
60 | if (ops & (1 << IPI_CALL_FUNC)) { |
61 | stats[IPI_CALL_FUNC]++; | |
99106986 | 62 | generic_smp_call_function_interrupt(); |
e6169c4b | 63 | } |
99106986 | 64 | |
e6169c4b GR |
65 | if (ops & (1 << IPI_IRQ_WORK)) { |
66 | stats[IPI_IRQ_WORK]++; | |
2c81b076 | 67 | irq_work_run(); |
e6169c4b | 68 | } |
2c81b076 | 69 | |
99106986 GR |
70 | BUG_ON((ops >> IPI_MAX) != 0); |
71 | } | |
72 | ||
73 | return IRQ_HANDLED; | |
74 | } | |
75 | ||
76 | static void (*send_arch_ipi)(const struct cpumask *mask); | |
77 | ||
78 | static int ipi_irq; | |
79 | void __init set_send_ipi(void (*func)(const struct cpumask *mask), int irq) | |
80 | { | |
81 | if (send_arch_ipi) | |
82 | return; | |
83 | ||
84 | send_arch_ipi = func; | |
85 | ipi_irq = irq; | |
86 | } | |
87 | ||
88 | static void | |
89 | send_ipi_message(const struct cpumask *to_whom, enum ipi_message_type operation) | |
90 | { | |
91 | int i; | |
92 | ||
93 | for_each_cpu(i, to_whom) | |
94 | set_bit(operation, &per_cpu_ptr(&ipi_data, i)->bits); | |
95 | ||
96 | smp_mb(); | |
97 | send_arch_ipi(to_whom); | |
98 | } | |
99 | ||
e6169c4b GR |
100 | static const char * const ipi_names[] = { |
101 | [IPI_EMPTY] = "Empty interrupts", | |
102 | [IPI_RESCHEDULE] = "Rescheduling interrupts", | |
103 | [IPI_CALL_FUNC] = "Function call interrupts", | |
104 | [IPI_IRQ_WORK] = "Irq work interrupts", | |
105 | }; | |
106 | ||
107 | int arch_show_interrupts(struct seq_file *p, int prec) | |
108 | { | |
109 | unsigned int cpu, i; | |
110 | ||
111 | for (i = 0; i < IPI_MAX; i++) { | |
112 | seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, | |
113 | prec >= 4 ? " " : ""); | |
114 | for_each_online_cpu(cpu) | |
115 | seq_printf(p, "%10lu ", | |
116 | per_cpu_ptr(&ipi_data, cpu)->stats[i]); | |
117 | seq_printf(p, " %s\n", ipi_names[i]); | |
118 | } | |
119 | ||
120 | return 0; | |
121 | } | |
122 | ||
99106986 GR |
123 | void arch_send_call_function_ipi_mask(struct cpumask *mask) |
124 | { | |
125 | send_ipi_message(mask, IPI_CALL_FUNC); | |
126 | } | |
127 | ||
128 | void arch_send_call_function_single_ipi(int cpu) | |
129 | { | |
130 | send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC); | |
131 | } | |
132 | ||
133 | static void ipi_stop(void *unused) | |
134 | { | |
135 | while (1); | |
136 | } | |
137 | ||
138 | void smp_send_stop(void) | |
139 | { | |
140 | on_each_cpu(ipi_stop, NULL, 1); | |
141 | } | |
142 | ||
143 | void smp_send_reschedule(int cpu) | |
144 | { | |
145 | send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE); | |
146 | } | |
147 | ||
2c81b076 GR |
148 | #ifdef CONFIG_IRQ_WORK |
149 | void arch_irq_work_raise(void) | |
150 | { | |
151 | send_ipi_message(cpumask_of(smp_processor_id()), IPI_IRQ_WORK); | |
152 | } | |
153 | #endif | |
154 | ||
99106986 GR |
155 | void __init smp_prepare_boot_cpu(void) |
156 | { | |
157 | } | |
158 | ||
159 | void __init smp_prepare_cpus(unsigned int max_cpus) | |
160 | { | |
161 | } | |
162 | ||
99106986 | 163 | static int ipi_dummy_dev; |
859e5f45 | 164 | |
99106986 GR |
165 | void __init setup_smp_ipi(void) |
166 | { | |
167 | int rc; | |
168 | ||
169 | if (ipi_irq == 0) | |
c9492737 | 170 | return; |
99106986 GR |
171 | |
172 | rc = request_percpu_irq(ipi_irq, handle_ipi, "IPI Interrupt", | |
173 | &ipi_dummy_dev); | |
174 | if (rc) | |
175 | panic("%s IRQ request failed\n", __func__); | |
176 | ||
859e5f45 | 177 | enable_percpu_irq(ipi_irq, 0); |
99106986 GR |
178 | } |
179 | ||
180 | void __init setup_smp(void) | |
181 | { | |
182 | struct device_node *node = NULL; | |
183 | int cpu; | |
184 | ||
398539dd | 185 | for_each_of_cpu_node(node) { |
99106986 GR |
186 | if (!of_device_is_available(node)) |
187 | continue; | |
188 | ||
189 | if (of_property_read_u32(node, "reg", &cpu)) | |
190 | continue; | |
191 | ||
192 | if (cpu >= NR_CPUS) | |
193 | continue; | |
194 | ||
195 | set_cpu_possible(cpu, true); | |
196 | set_cpu_present(cpu, true); | |
197 | } | |
198 | } | |
199 | ||
200 | extern void _start_smp_secondary(void); | |
201 | ||
202 | volatile unsigned int secondary_hint; | |
8077e66b | 203 | volatile unsigned int secondary_hint2; |
99106986 GR |
204 | volatile unsigned int secondary_ccr; |
205 | volatile unsigned int secondary_stack; | |
206 | ||
aefd9461 GR |
207 | unsigned long secondary_msa1; |
208 | ||
99106986 GR |
209 | int __cpu_up(unsigned int cpu, struct task_struct *tidle) |
210 | { | |
859e5f45 | 211 | unsigned long mask = 1 << cpu; |
99106986 | 212 | |
0f231dcf GR |
213 | secondary_stack = |
214 | (unsigned int) task_stack_page(tidle) + THREAD_SIZE - 8; | |
99106986 | 215 | secondary_hint = mfcr("cr31"); |
8077e66b | 216 | secondary_hint2 = mfcr("cr<21, 1>"); |
99106986 | 217 | secondary_ccr = mfcr("cr18"); |
aefd9461 | 218 | secondary_msa1 = read_mmu_msa1(); |
99106986 GR |
219 | |
220 | /* | |
221 | * Because other CPUs are in reset status, we must flush data | |
222 | * from cache to out and secondary CPUs use them in | |
223 | * csky_start_secondary(void) | |
224 | */ | |
225 | mtcr("cr17", 0x22); | |
226 | ||
859e5f45 GR |
227 | if (mask & mfcr("cr<29, 0>")) { |
228 | send_arch_ipi(cpumask_of(cpu)); | |
229 | } else { | |
230 | /* Enable cpu in SMP reset ctrl reg */ | |
231 | mask |= mfcr("cr<29, 0>"); | |
232 | mtcr("cr<29, 0>", mask); | |
233 | } | |
99106986 GR |
234 | |
235 | /* Wait for the cpu online */ | |
236 | while (!cpu_online(cpu)); | |
237 | ||
238 | secondary_stack = 0; | |
239 | ||
240 | return 0; | |
241 | } | |
242 | ||
243 | void __init smp_cpus_done(unsigned int max_cpus) | |
244 | { | |
245 | } | |
246 | ||
247 | int setup_profiling_timer(unsigned int multiplier) | |
248 | { | |
249 | return -EINVAL; | |
250 | } | |
251 | ||
252 | void csky_start_secondary(void) | |
253 | { | |
254 | struct mm_struct *mm = &init_mm; | |
255 | unsigned int cpu = smp_processor_id(); | |
256 | ||
257 | mtcr("cr31", secondary_hint); | |
8077e66b | 258 | mtcr("cr<21, 1>", secondary_hint2); |
99106986 GR |
259 | mtcr("cr18", secondary_ccr); |
260 | ||
261 | mtcr("vbr", vec_base); | |
262 | ||
263 | flush_tlb_all(); | |
264 | write_mmu_pagemask(0); | |
265 | TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir); | |
266 | TLBMISS_HANDLER_SETUP_PGD_KERNEL(swapper_pg_dir); | |
267 | ||
99106986 GR |
268 | #ifdef CONFIG_CPU_HAS_FPU |
269 | init_fpu(); | |
270 | #endif | |
271 | ||
859e5f45 | 272 | enable_percpu_irq(ipi_irq, 0); |
99106986 GR |
273 | |
274 | mmget(mm); | |
275 | mmgrab(mm); | |
276 | current->active_mm = mm; | |
277 | cpumask_set_cpu(cpu, mm_cpumask(mm)); | |
278 | ||
279 | notify_cpu_starting(cpu); | |
280 | set_cpu_online(cpu, true); | |
281 | ||
282 | pr_info("CPU%u Online: %s...\n", cpu, __func__); | |
283 | ||
284 | local_irq_enable(); | |
285 | preempt_disable(); | |
286 | cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); | |
287 | } | |
859e5f45 GR |
288 | |
289 | #ifdef CONFIG_HOTPLUG_CPU | |
290 | int __cpu_disable(void) | |
291 | { | |
292 | unsigned int cpu = smp_processor_id(); | |
293 | ||
294 | set_cpu_online(cpu, false); | |
295 | ||
296 | irq_migrate_all_off_this_cpu(); | |
297 | ||
298 | clear_tasks_mm_cpumask(cpu); | |
299 | ||
300 | return 0; | |
301 | } | |
302 | ||
303 | void __cpu_die(unsigned int cpu) | |
304 | { | |
305 | if (!cpu_wait_death(cpu, 5)) { | |
306 | pr_crit("CPU%u: shutdown failed\n", cpu); | |
307 | return; | |
308 | } | |
309 | pr_notice("CPU%u: shutdown\n", cpu); | |
310 | } | |
311 | ||
312 | void arch_cpu_idle_dead(void) | |
313 | { | |
314 | idle_task_exit(); | |
315 | ||
316 | cpu_report_death(); | |
317 | ||
318 | while (!secondary_stack) | |
319 | arch_cpu_idle(); | |
320 | ||
321 | local_irq_disable(); | |
322 | ||
323 | asm volatile( | |
324 | "mov sp, %0\n" | |
325 | "mov r8, %0\n" | |
326 | "jmpi csky_start_secondary" | |
327 | : | |
328 | : "r" (secondary_stack)); | |
329 | } | |
330 | #endif |