]>
Commit | Line | Data |
---|---|---|
ccf55117 AK |
1 | /* |
2 | * Copyright (c) 2014 Samsung Electronics Co., Ltd. | |
3 | * http://www.samsung.com | |
4 | * | |
5 | * arch/arm/mach-exynos/mcpm-exynos.c | |
6 | * | |
7 | * Based on arch/arm/mach-vexpress/dcscb.c | |
8 | * | |
9 | * This program is free software; you can redistribute it and/or modify | |
10 | * it under the terms of the GNU General Public License version 2 as | |
11 | * published by the Free Software Foundation. | |
12 | */ | |
13 | ||
14 | #include <linux/arm-cci.h> | |
15 | #include <linux/delay.h> | |
16 | #include <linux/io.h> | |
17 | #include <linux/of_address.h> | |
18 | ||
19 | #include <asm/cputype.h> | |
20 | #include <asm/cp15.h> | |
21 | #include <asm/mcpm.h> | |
22 | ||
23 | #include "regs-pmu.h" | |
24 | #include "common.h" | |
25 | ||
26 | #define EXYNOS5420_CPUS_PER_CLUSTER 4 | |
27 | #define EXYNOS5420_NR_CLUSTERS 2 | |
ccf55117 | 28 | |
20fe6f98 AK |
29 | #define EXYNOS5420_ENABLE_AUTOMATIC_CORE_DOWN BIT(9) |
30 | #define EXYNOS5420_USE_ARM_CORE_DOWN_STATE BIT(29) | |
31 | #define EXYNOS5420_USE_L2_COMMON_UP_STATE BIT(30) | |
32 | ||
ccf55117 AK |
33 | /* |
34 | * The common v7_exit_coherency_flush API could not be used because of the | |
35 | * Erratum 799270 workaround. This macro is the same as the common one (in | |
36 | * arch/arm/include/asm/cacheflush.h) except for the erratum handling. | |
37 | */ | |
38 | #define exynos_v7_exit_coherency_flush(level) \ | |
39 | asm volatile( \ | |
40 | "stmfd sp!, {fp, ip}\n\t"\ | |
41 | "mrc p15, 0, r0, c1, c0, 0 @ get SCTLR\n\t" \ | |
42 | "bic r0, r0, #"__stringify(CR_C)"\n\t" \ | |
43 | "mcr p15, 0, r0, c1, c0, 0 @ set SCTLR\n\t" \ | |
44 | "isb\n\t"\ | |
45 | "bl v7_flush_dcache_"__stringify(level)"\n\t" \ | |
ccf55117 AK |
46 | "mrc p15, 0, r0, c1, c0, 1 @ get ACTLR\n\t" \ |
47 | "bic r0, r0, #(1 << 6) @ disable local coherency\n\t" \ | |
48 | /* Dummy Load of a device register to avoid Erratum 799270 */ \ | |
49 | "ldr r4, [%0]\n\t" \ | |
50 | "and r4, r4, #0\n\t" \ | |
51 | "orr r0, r0, r4\n\t" \ | |
52 | "mcr p15, 0, r0, c1, c0, 1 @ set ACTLR\n\t" \ | |
53 | "isb\n\t" \ | |
54 | "dsb\n\t" \ | |
55 | "ldmfd sp!, {fp, ip}" \ | |
56 | : \ | |
2e94ac42 | 57 | : "Ir" (pmu_base_addr + S5P_INFORM0) \ |
ccf55117 AK |
58 | : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", \ |
59 | "r9", "r10", "lr", "memory") | |
60 | ||
61 | /* | |
62 | * We can't use regular spinlocks. In the switcher case, it is possible | |
63 | * for an outbound CPU to call power_down() after its inbound counterpart | |
64 | * is already live using the same logical CPU number which trips lockdep | |
65 | * debugging. | |
66 | */ | |
67 | static arch_spinlock_t exynos_mcpm_lock = __ARCH_SPIN_LOCK_UNLOCKED; | |
68 | static int | |
69 | cpu_use_count[EXYNOS5420_CPUS_PER_CLUSTER][EXYNOS5420_NR_CLUSTERS]; | |
70 | ||
71 | #define exynos_cluster_usecnt(cluster) \ | |
72 | (cpu_use_count[0][cluster] + \ | |
73 | cpu_use_count[1][cluster] + \ | |
74 | cpu_use_count[2][cluster] + \ | |
75 | cpu_use_count[3][cluster]) | |
76 | ||
77 | #define exynos_cluster_unused(cluster) !exynos_cluster_usecnt(cluster) | |
78 | ||
ccf55117 AK |
79 | static int exynos_power_up(unsigned int cpu, unsigned int cluster) |
80 | { | |
81 | unsigned int cpunr = cpu + (cluster * EXYNOS5420_CPUS_PER_CLUSTER); | |
ccf55117 AK |
82 | |
83 | pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster); | |
84 | if (cpu >= EXYNOS5420_CPUS_PER_CLUSTER || | |
85 | cluster >= EXYNOS5420_NR_CLUSTERS) | |
86 | return -EINVAL; | |
87 | ||
88 | /* | |
89 | * Since this is called with IRQs enabled, and no arch_spin_lock_irq | |
90 | * variant exists, we need to disable IRQs manually here. | |
91 | */ | |
92 | local_irq_disable(); | |
93 | arch_spin_lock(&exynos_mcpm_lock); | |
94 | ||
95 | cpu_use_count[cpu][cluster]++; | |
96 | if (cpu_use_count[cpu][cluster] == 1) { | |
97 | bool was_cluster_down = | |
98 | (exynos_cluster_usecnt(cluster) == 1); | |
99 | ||
100 | /* | |
101 | * Turn on the cluster (L2/COMMON) and then power on the | |
102 | * cores. | |
103 | */ | |
104 | if (was_cluster_down) | |
20fe6f98 | 105 | exynos_cluster_power_up(cluster); |
ccf55117 | 106 | |
20fe6f98 | 107 | exynos_cpu_power_up(cpunr); |
ccf55117 AK |
108 | } else if (cpu_use_count[cpu][cluster] != 2) { |
109 | /* | |
110 | * The only possible values are: | |
111 | * 0 = CPU down | |
112 | * 1 = CPU (still) up | |
113 | * 2 = CPU requested to be up before it had a chance | |
114 | * to actually make itself down. | |
115 | * Any other value is a bug. | |
116 | */ | |
117 | BUG(); | |
118 | } | |
119 | ||
120 | arch_spin_unlock(&exynos_mcpm_lock); | |
121 | local_irq_enable(); | |
122 | ||
20fe6f98 | 123 | return 0; |
ccf55117 AK |
124 | } |
125 | ||
126 | /* | |
127 | * NOTE: This function requires the stack data to be visible through power down | |
128 | * and can only be executed on processors like A15 and A7 that hit the cache | |
129 | * with the C bit clear in the SCTLR register. | |
130 | */ | |
131 | static void exynos_power_down(void) | |
132 | { | |
133 | unsigned int mpidr, cpu, cluster; | |
134 | bool last_man = false, skip_wfi = false; | |
135 | unsigned int cpunr; | |
136 | ||
137 | mpidr = read_cpuid_mpidr(); | |
138 | cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); | |
139 | cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); | |
140 | cpunr = cpu + (cluster * EXYNOS5420_CPUS_PER_CLUSTER); | |
141 | ||
142 | pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster); | |
143 | BUG_ON(cpu >= EXYNOS5420_CPUS_PER_CLUSTER || | |
144 | cluster >= EXYNOS5420_NR_CLUSTERS); | |
145 | ||
146 | __mcpm_cpu_going_down(cpu, cluster); | |
147 | ||
148 | arch_spin_lock(&exynos_mcpm_lock); | |
149 | BUG_ON(__mcpm_cluster_state(cluster) != CLUSTER_UP); | |
150 | cpu_use_count[cpu][cluster]--; | |
151 | if (cpu_use_count[cpu][cluster] == 0) { | |
152 | exynos_cpu_power_down(cpunr); | |
153 | ||
20fe6f98 AK |
154 | if (exynos_cluster_unused(cluster)) { |
155 | exynos_cluster_power_down(cluster); | |
ccf55117 | 156 | last_man = true; |
20fe6f98 | 157 | } |
ccf55117 AK |
158 | } else if (cpu_use_count[cpu][cluster] == 1) { |
159 | /* | |
160 | * A power_up request went ahead of us. | |
161 | * Even if we do not want to shut this CPU down, | |
162 | * the caller expects a certain state as if the WFI | |
163 | * was aborted. So let's continue with cache cleaning. | |
164 | */ | |
165 | skip_wfi = true; | |
166 | } else { | |
167 | BUG(); | |
168 | } | |
169 | ||
170 | if (last_man && __mcpm_outbound_enter_critical(cpu, cluster)) { | |
171 | arch_spin_unlock(&exynos_mcpm_lock); | |
172 | ||
af040ffc | 173 | if (read_cpuid_part() == ARM_CPU_PART_CORTEX_A15) { |
ccf55117 AK |
174 | /* |
175 | * On the Cortex-A15 we need to disable | |
176 | * L2 prefetching before flushing the cache. | |
177 | */ | |
178 | asm volatile( | |
179 | "mcr p15, 1, %0, c15, c0, 3\n\t" | |
180 | "isb\n\t" | |
181 | "dsb" | |
182 | : : "r" (0x400)); | |
183 | } | |
184 | ||
185 | /* Flush all cache levels for this cluster. */ | |
186 | exynos_v7_exit_coherency_flush(all); | |
187 | ||
188 | /* | |
189 | * Disable cluster-level coherency by masking | |
190 | * incoming snoops and DVM messages: | |
191 | */ | |
192 | cci_disable_port_by_cpu(mpidr); | |
193 | ||
194 | __mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN); | |
195 | } else { | |
196 | arch_spin_unlock(&exynos_mcpm_lock); | |
197 | ||
198 | /* Disable and flush the local CPU cache. */ | |
199 | exynos_v7_exit_coherency_flush(louis); | |
200 | } | |
201 | ||
202 | __mcpm_cpu_down(cpu, cluster); | |
203 | ||
204 | /* Now we are prepared for power-down, do it: */ | |
205 | if (!skip_wfi) | |
206 | wfi(); | |
207 | ||
208 | /* Not dead at this point? Let our caller cope. */ | |
209 | } | |
210 | ||
7c5688e7 | 211 | static int exynos_wait_for_powerdown(unsigned int cpu, unsigned int cluster) |
ccf55117 AK |
212 | { |
213 | unsigned int tries = 100; | |
214 | unsigned int cpunr = cpu + (cluster * EXYNOS5420_CPUS_PER_CLUSTER); | |
215 | ||
216 | pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster); | |
217 | BUG_ON(cpu >= EXYNOS5420_CPUS_PER_CLUSTER || | |
218 | cluster >= EXYNOS5420_NR_CLUSTERS); | |
219 | ||
220 | /* Wait for the core state to be OFF */ | |
221 | while (tries--) { | |
222 | if (ACCESS_ONCE(cpu_use_count[cpu][cluster]) == 0) { | |
223 | if ((exynos_cpu_power_state(cpunr) == 0)) | |
224 | return 0; /* success: the CPU is halted */ | |
225 | } | |
226 | ||
227 | /* Otherwise, wait and retry: */ | |
228 | msleep(1); | |
229 | } | |
230 | ||
231 | return -ETIMEDOUT; /* timeout */ | |
232 | } | |
233 | ||
fc2cac41 CK |
234 | static void exynos_powered_up(void) |
235 | { | |
236 | unsigned int mpidr, cpu, cluster; | |
237 | ||
238 | mpidr = read_cpuid_mpidr(); | |
239 | cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); | |
240 | cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); | |
241 | ||
242 | arch_spin_lock(&exynos_mcpm_lock); | |
243 | if (cpu_use_count[cpu][cluster] == 0) | |
244 | cpu_use_count[cpu][cluster] = 1; | |
245 | arch_spin_unlock(&exynos_mcpm_lock); | |
246 | } | |
247 | ||
248 | static void exynos_suspend(u64 residency) | |
249 | { | |
250 | unsigned int mpidr, cpunr; | |
251 | ||
252 | exynos_power_down(); | |
253 | ||
254 | /* | |
255 | * Execution reaches here only if cpu did not power down. | |
256 | * Hence roll back the changes done in exynos_power_down function. | |
257 | * | |
258 | * CAUTION: "This function requires the stack data to be visible through | |
259 | * power down and can only be executed on processors like A15 and A7 | |
260 | * that hit the cache with the C bit clear in the SCTLR register." | |
261 | */ | |
262 | mpidr = read_cpuid_mpidr(); | |
263 | cpunr = exynos_pmu_cpunr(mpidr); | |
264 | ||
265 | exynos_cpu_power_up(cpunr); | |
266 | } | |
267 | ||
ccf55117 AK |
268 | static const struct mcpm_platform_ops exynos_power_ops = { |
269 | .power_up = exynos_power_up, | |
270 | .power_down = exynos_power_down, | |
7c5688e7 | 271 | .wait_for_powerdown = exynos_wait_for_powerdown, |
fc2cac41 CK |
272 | .suspend = exynos_suspend, |
273 | .powered_up = exynos_powered_up, | |
ccf55117 AK |
274 | }; |
275 | ||
276 | static void __init exynos_mcpm_usage_count_init(void) | |
277 | { | |
278 | unsigned int mpidr, cpu, cluster; | |
279 | ||
280 | mpidr = read_cpuid_mpidr(); | |
281 | cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); | |
282 | cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); | |
283 | ||
284 | pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster); | |
285 | BUG_ON(cpu >= EXYNOS5420_CPUS_PER_CLUSTER || | |
286 | cluster >= EXYNOS5420_NR_CLUSTERS); | |
287 | ||
288 | cpu_use_count[cpu][cluster] = 1; | |
289 | } | |
290 | ||
291 | /* | |
292 | * Enable cluster-level coherency, in preparation for turning on the MMU. | |
293 | */ | |
294 | static void __naked exynos_pm_power_up_setup(unsigned int affinity_level) | |
295 | { | |
296 | asm volatile ("\n" | |
297 | "cmp r0, #1\n" | |
298 | "bxne lr\n" | |
299 | "b cci_enable_port_for_self"); | |
300 | } | |
301 | ||
fbb04990 NP |
302 | static void __init exynos_cache_off(void) |
303 | { | |
af040ffc | 304 | if (read_cpuid_part() == ARM_CPU_PART_CORTEX_A15) { |
fbb04990 NP |
305 | /* disable L2 prefetching on the Cortex-A15 */ |
306 | asm volatile( | |
307 | "mcr p15, 1, %0, c15, c0, 3\n\t" | |
308 | "isb\n\t" | |
309 | "dsb" | |
310 | : : "r" (0x400)); | |
311 | } | |
312 | exynos_v7_exit_coherency_flush(all); | |
313 | } | |
314 | ||
f99acff1 AK |
315 | static const struct of_device_id exynos_dt_mcpm_match[] = { |
316 | { .compatible = "samsung,exynos5420" }, | |
317 | { .compatible = "samsung,exynos5800" }, | |
318 | {}, | |
319 | }; | |
320 | ||
ccf55117 AK |
321 | static int __init exynos_mcpm_init(void) |
322 | { | |
323 | struct device_node *node; | |
324 | void __iomem *ns_sram_base_addr; | |
20fe6f98 | 325 | unsigned int value, i; |
ccf55117 AK |
326 | int ret; |
327 | ||
f99acff1 | 328 | node = of_find_matching_node(NULL, exynos_dt_mcpm_match); |
ccf55117 AK |
329 | if (!node) |
330 | return -ENODEV; | |
331 | of_node_put(node); | |
332 | ||
333 | if (!cci_probed()) | |
334 | return -ENODEV; | |
335 | ||
336 | node = of_find_compatible_node(NULL, NULL, | |
337 | "samsung,exynos4210-sysram-ns"); | |
338 | if (!node) | |
339 | return -ENODEV; | |
340 | ||
341 | ns_sram_base_addr = of_iomap(node, 0); | |
342 | of_node_put(node); | |
343 | if (!ns_sram_base_addr) { | |
344 | pr_err("failed to map non-secure iRAM base address\n"); | |
345 | return -ENOMEM; | |
346 | } | |
347 | ||
348 | /* | |
349 | * To increase the stability of KFC reset we need to program | |
350 | * the PMU SPARE3 register | |
351 | */ | |
2e94ac42 | 352 | pmu_raw_writel(EXYNOS5420_SWRESET_KFC_SEL, S5P_PMU_SPARE3); |
ccf55117 AK |
353 | |
354 | exynos_mcpm_usage_count_init(); | |
355 | ||
356 | ret = mcpm_platform_register(&exynos_power_ops); | |
357 | if (!ret) | |
358 | ret = mcpm_sync_init(exynos_pm_power_up_setup); | |
fbb04990 NP |
359 | if (!ret) |
360 | ret = mcpm_loopback(exynos_cache_off); /* turn on the CCI */ | |
ccf55117 AK |
361 | if (ret) { |
362 | iounmap(ns_sram_base_addr); | |
363 | return ret; | |
364 | } | |
365 | ||
366 | mcpm_smp_set_ops(); | |
367 | ||
368 | pr_info("Exynos MCPM support installed\n"); | |
369 | ||
20fe6f98 AK |
370 | /* |
371 | * On Exynos5420/5800 for the A15 and A7 clusters: | |
372 | * | |
373 | * EXYNOS5420_ENABLE_AUTOMATIC_CORE_DOWN ensures that all the cores | |
374 | * in a cluster are turned off before turning off the cluster L2. | |
375 | * | |
376 | * EXYNOS5420_USE_ARM_CORE_DOWN_STATE ensures that a cores is powered | |
377 | * off before waking it up. | |
378 | * | |
379 | * EXYNOS5420_USE_L2_COMMON_UP_STATE ensures that cluster L2 will be | |
380 | * turned on before the first man is powered up. | |
381 | */ | |
382 | for (i = 0; i < EXYNOS5420_NR_CLUSTERS; i++) { | |
2e94ac42 | 383 | value = pmu_raw_readl(EXYNOS_COMMON_OPTION(i)); |
20fe6f98 AK |
384 | value |= EXYNOS5420_ENABLE_AUTOMATIC_CORE_DOWN | |
385 | EXYNOS5420_USE_ARM_CORE_DOWN_STATE | | |
386 | EXYNOS5420_USE_L2_COMMON_UP_STATE; | |
2e94ac42 | 387 | pmu_raw_writel(value, EXYNOS_COMMON_OPTION(i)); |
20fe6f98 AK |
388 | } |
389 | ||
ccf55117 | 390 | /* |
7cbcb9d4 DA |
391 | * U-Boot SPL is hardcoded to jump to the start of ns_sram_base_addr |
392 | * as part of secondary_cpu_start(). Let's redirect it to the | |
393 | * mcpm_entry_point(). | |
ccf55117 | 394 | */ |
7cbcb9d4 DA |
395 | __raw_writel(0xe59f0000, ns_sram_base_addr); /* ldr r0, [pc, #0] */ |
396 | __raw_writel(0xe12fff10, ns_sram_base_addr + 4); /* bx r0 */ | |
397 | __raw_writel(virt_to_phys(mcpm_entry_point), ns_sram_base_addr + 8); | |
ccf55117 AK |
398 | |
399 | iounmap(ns_sram_base_addr); | |
400 | ||
401 | return ret; | |
402 | } | |
403 | ||
404 | early_initcall(exynos_mcpm_init); |