]>
Commit | Line | Data |
---|---|---|
9cce7a43 CM |
1 | /* |
2 | * Based on arch/arm/mm/proc.S | |
3 | * | |
4 | * Copyright (C) 2001 Deep Blue Solutions Ltd. | |
5 | * Copyright (C) 2012 ARM Ltd. | |
6 | * Author: Catalin Marinas <catalin.marinas@arm.com> | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or modify | |
9 | * it under the terms of the GNU General Public License version 2 as | |
10 | * published by the Free Software Foundation. | |
11 | * | |
12 | * This program is distributed in the hope that it will be useful, | |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | * GNU General Public License for more details. | |
16 | * | |
17 | * You should have received a copy of the GNU General Public License | |
18 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
19 | */ | |
20 | ||
21 | #include <linux/init.h> | |
22 | #include <linux/linkage.h> | |
23 | #include <asm/assembler.h> | |
24 | #include <asm/asm-offsets.h> | |
25 | #include <asm/hwcap.h> | |
9cce7a43 | 26 | #include <asm/pgtable.h> |
cabe1c81 | 27 | #include <asm/pgtable-hwdef.h> |
104a0c02 AP |
28 | #include <asm/cpufeature.h> |
29 | #include <asm/alternative.h> | |
9cce7a43 | 30 | |
35a86976 CM |
31 | #ifdef CONFIG_ARM64_64K_PAGES |
32 | #define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K | |
44eaacf1 SP |
33 | #elif defined(CONFIG_ARM64_16K_PAGES) |
34 | #define TCR_TG_FLAGS TCR_TG0_16K | TCR_TG1_16K | |
35 | #else /* CONFIG_ARM64_4K_PAGES */ | |
35a86976 CM |
36 | #define TCR_TG_FLAGS TCR_TG0_4K | TCR_TG1_4K |
37 | #endif | |
38 | ||
35a86976 | 39 | #define TCR_SMP_FLAGS TCR_SHARED |
9cce7a43 | 40 | |
35a86976 CM |
41 | /* PTWs cacheable, inner/outer WBWA */ |
42 | #define TCR_CACHE_FLAGS TCR_IRGN_WBWA | TCR_ORGN_WBWA | |
43 | ||
9cce7a43 CM |
44 | #define MAIR(attr, mt) ((attr) << ((mt) * 8)) |
45 | ||
9cce7a43 CM |
46 | /* |
47 | * cpu_do_idle() | |
48 | * | |
49 | * Idle the processor (wait for interrupt). | |
50 | */ | |
51 | ENTRY(cpu_do_idle) | |
52 | dsb sy // WFI may enter a low-power mode | |
53 | wfi | |
54 | ret | |
55 | ENDPROC(cpu_do_idle) | |
56 | ||
af3cfdbf | 57 | #ifdef CONFIG_CPU_PM |
6732bc65 LP |
58 | /** |
59 | * cpu_do_suspend - save CPU registers context | |
60 | * | |
61 | * x0: virtual address of context pointer | |
62 | */ | |
63 | ENTRY(cpu_do_suspend) | |
64 | mrs x2, tpidr_el0 | |
65 | mrs x3, tpidrro_el0 | |
66 | mrs x4, contextidr_el1 | |
cabe1c81 JM |
67 | mrs x5, cpacr_el1 |
68 | mrs x6, tcr_el1 | |
69 | mrs x7, vbar_el1 | |
70 | mrs x8, mdscr_el1 | |
71 | mrs x9, oslsr_el1 | |
72 | mrs x10, sctlr_el1 | |
623b476f MR |
73 | mrs x11, tpidr_el1 |
74 | mrs x12, sp_el0 | |
6732bc65 | 75 | stp x2, x3, [x0] |
cabe1c81 JM |
76 | stp x4, xzr, [x0, #16] |
77 | stp x5, x6, [x0, #32] | |
78 | stp x7, x8, [x0, #48] | |
79 | stp x9, x10, [x0, #64] | |
623b476f | 80 | stp x11, x12, [x0, #80] |
6732bc65 LP |
81 | ret |
82 | ENDPROC(cpu_do_suspend) | |
83 | ||
84 | /** | |
85 | * cpu_do_resume - restore CPU register context | |
86 | * | |
cabe1c81 | 87 | * x0: Address of context pointer |
6732bc65 | 88 | */ |
b159a095 | 89 | .pushsection ".idmap.text", "awx" |
6732bc65 | 90 | ENTRY(cpu_do_resume) |
6732bc65 LP |
91 | ldp x2, x3, [x0] |
92 | ldp x4, x5, [x0, #16] | |
cabe1c81 JM |
93 | ldp x6, x8, [x0, #32] |
94 | ldp x9, x10, [x0, #48] | |
95 | ldp x11, x12, [x0, #64] | |
623b476f | 96 | ldp x13, x14, [x0, #80] |
6732bc65 LP |
97 | msr tpidr_el0, x2 |
98 | msr tpidrro_el0, x3 | |
99 | msr contextidr_el1, x4 | |
6732bc65 | 100 | msr cpacr_el1, x6 |
cabe1c81 JM |
101 | |
102 | /* Don't change t0sz here, mask those bits when restoring */ | |
103 | mrs x5, tcr_el1 | |
104 | bfi x8, x5, TCR_T0SZ_OFFSET, TCR_TxSZ_WIDTH | |
105 | ||
6732bc65 LP |
106 | msr tcr_el1, x8 |
107 | msr vbar_el1, x9 | |
744c6c37 JM |
108 | |
109 | /* | |
110 | * __cpu_setup() cleared MDSCR_EL1.MDE and friends, before unmasking | |
111 | * debug exceptions. By restoring MDSCR_EL1 here, we may take a debug | |
0fbeb318 | 112 | * exception. Mask them until local_daif_restore() in cpu_suspend() |
744c6c37 JM |
113 | * resets them. |
114 | */ | |
0fbeb318 | 115 | disable_daif |
6732bc65 | 116 | msr mdscr_el1, x10 |
744c6c37 | 117 | |
cabe1c81 | 118 | msr sctlr_el1, x12 |
623b476f MR |
119 | msr tpidr_el1, x13 |
120 | msr sp_el0, x14 | |
6732bc65 LP |
121 | /* |
122 | * Restore oslsr_el1 by writing oslar_el1 | |
123 | */ | |
124 | ubfx x11, x11, #1, #1 | |
125 | msr oslar_el1, x11 | |
f436b2ac | 126 | reset_pmuserenr_el0 x0 // Disable PMU access from EL0 |
6732bc65 LP |
127 | isb |
128 | ret | |
129 | ENDPROC(cpu_do_resume) | |
b6113038 | 130 | .popsection |
6732bc65 LP |
131 | #endif |
132 | ||
9cce7a43 | 133 | /* |
812944e9 | 134 | * cpu_do_switch_mm(pgd_phys, tsk) |
9cce7a43 CM |
135 | * |
136 | * Set the translation table base pointer to be pgd_phys. | |
137 | * | |
138 | * - pgd_phys - physical address of new TTB | |
139 | */ | |
140 | ENTRY(cpu_do_switch_mm) | |
c4f25e12 | 141 | mrs x2, ttbr1_el1 |
5aec715d | 142 | mmid x1, x1 // get mm->context.id |
6884bbc7 CM |
143 | #ifdef CONFIG_ARM64_SW_TTBR0_PAN |
144 | bfi x0, x1, #48, #16 // set the ASID field in TTBR0 | |
145 | #endif | |
c4f25e12 WD |
146 | bfi x2, x1, #48, #16 // set the ASID |
147 | msr ttbr1_el1, x2 // in TTBR1 (since TCR.A1 is set) | |
148 | isb | |
149 | msr ttbr0_el1, x0 // now update TTBR0 | |
9cce7a43 | 150 | isb |
aafb2daf | 151 | b post_ttbr_update_workaround // Back to C code... |
9cce7a43 CM |
152 | ENDPROC(cpu_do_switch_mm) |
153 | ||
b159a095 | 154 | .pushsection ".idmap.text", "awx" |
5c10f9ca WD |
155 | |
156 | .macro __idmap_cpu_set_reserved_ttbr1, tmp1, tmp2 | |
157 | adrp \tmp1, empty_zero_page | |
158 | msr ttbr1_el1, \tmp2 | |
159 | isb | |
160 | tlbi vmalle1 | |
161 | dsb nsh | |
162 | isb | |
163 | .endm | |
164 | ||
50e1881d MR |
165 | /* |
166 | * void idmap_cpu_replace_ttbr1(phys_addr_t new_pgd) | |
167 | * | |
168 | * This is the low-level counterpart to cpu_replace_ttbr1, and should not be | |
169 | * called by anything else. It can only be executed from a TTBR0 mapping. | |
170 | */ | |
171 | ENTRY(idmap_cpu_replace_ttbr1) | |
0fbeb318 | 172 | save_and_disable_daif flags=x2 |
50e1881d | 173 | |
5c10f9ca | 174 | __idmap_cpu_set_reserved_ttbr1 x1, x3 |
50e1881d MR |
175 | |
176 | msr ttbr1_el1, x0 | |
177 | isb | |
178 | ||
0fbeb318 | 179 | restore_daif x2 |
50e1881d MR |
180 | |
181 | ret | |
182 | ENDPROC(idmap_cpu_replace_ttbr1) | |
183 | .popsection | |
184 | ||
5c10f9ca | 185 | #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 |
b159a095 | 186 | .pushsection ".idmap.text", "awx" |
5c10f9ca WD |
187 | |
188 | .macro __idmap_kpti_get_pgtable_ent, type | |
189 | dc cvac, cur_\()\type\()p // Ensure any existing dirty | |
190 | dmb sy // lines are written back before | |
191 | ldr \type, [cur_\()\type\()p] // loading the entry | |
192 | tbz \type, #0, next_\()\type // Skip invalid entries | |
193 | .endm | |
194 | ||
195 | .macro __idmap_kpti_put_pgtable_ent_ng, type | |
196 | orr \type, \type, #PTE_NG // Same bit for blocks and pages | |
197 | str \type, [cur_\()\type\()p] // Update the entry and ensure it | |
198 | dc civac, cur_\()\type\()p // is visible to all CPUs. | |
199 | .endm | |
200 | ||
201 | /* | |
202 | * void __kpti_install_ng_mappings(int cpu, int num_cpus, phys_addr_t swapper) | |
203 | * | |
204 | * Called exactly once from stop_machine context by each CPU found during boot. | |
205 | */ | |
206 | __idmap_kpti_flag: | |
207 | .long 1 | |
208 | ENTRY(idmap_kpti_install_ng_mappings) | |
209 | cpu .req w0 | |
210 | num_cpus .req w1 | |
211 | swapper_pa .req x2 | |
212 | swapper_ttb .req x3 | |
213 | flag_ptr .req x4 | |
214 | cur_pgdp .req x5 | |
215 | end_pgdp .req x6 | |
216 | pgd .req x7 | |
217 | cur_pudp .req x8 | |
218 | end_pudp .req x9 | |
219 | pud .req x10 | |
220 | cur_pmdp .req x11 | |
221 | end_pmdp .req x12 | |
222 | pmd .req x13 | |
223 | cur_ptep .req x14 | |
224 | end_ptep .req x15 | |
225 | pte .req x16 | |
226 | ||
227 | mrs swapper_ttb, ttbr1_el1 | |
228 | adr flag_ptr, __idmap_kpti_flag | |
229 | ||
230 | cbnz cpu, __idmap_kpti_secondary | |
231 | ||
232 | /* We're the boot CPU. Wait for the others to catch up */ | |
233 | sevl | |
234 | 1: wfe | |
235 | ldaxr w18, [flag_ptr] | |
236 | eor w18, w18, num_cpus | |
237 | cbnz w18, 1b | |
238 | ||
239 | /* We need to walk swapper, so turn off the MMU. */ | |
240 | pre_disable_mmu_workaround | |
241 | mrs x18, sctlr_el1 | |
242 | bic x18, x18, #SCTLR_ELx_M | |
243 | msr sctlr_el1, x18 | |
244 | isb | |
245 | ||
246 | /* Everybody is enjoying the idmap, so we can rewrite swapper. */ | |
247 | /* PGD */ | |
248 | mov cur_pgdp, swapper_pa | |
249 | add end_pgdp, cur_pgdp, #(PTRS_PER_PGD * 8) | |
250 | do_pgd: __idmap_kpti_get_pgtable_ent pgd | |
251 | tbnz pgd, #1, walk_puds | |
252 | __idmap_kpti_put_pgtable_ent_ng pgd | |
253 | next_pgd: | |
254 | add cur_pgdp, cur_pgdp, #8 | |
255 | cmp cur_pgdp, end_pgdp | |
256 | b.ne do_pgd | |
257 | ||
258 | /* Publish the updated tables and nuke all the TLBs */ | |
259 | dsb sy | |
260 | tlbi vmalle1is | |
261 | dsb ish | |
262 | isb | |
263 | ||
264 | /* We're done: fire up the MMU again */ | |
265 | mrs x18, sctlr_el1 | |
266 | orr x18, x18, #SCTLR_ELx_M | |
267 | msr sctlr_el1, x18 | |
268 | isb | |
269 | ||
270 | /* Set the flag to zero to indicate that we're all done */ | |
271 | str wzr, [flag_ptr] | |
272 | ret | |
273 | ||
274 | /* PUD */ | |
275 | walk_puds: | |
276 | .if CONFIG_PGTABLE_LEVELS > 3 | |
277 | pte_to_phys cur_pudp, pgd | |
278 | add end_pudp, cur_pudp, #(PTRS_PER_PUD * 8) | |
279 | do_pud: __idmap_kpti_get_pgtable_ent pud | |
280 | tbnz pud, #1, walk_pmds | |
281 | __idmap_kpti_put_pgtable_ent_ng pud | |
282 | next_pud: | |
283 | add cur_pudp, cur_pudp, 8 | |
284 | cmp cur_pudp, end_pudp | |
285 | b.ne do_pud | |
286 | b next_pgd | |
287 | .else /* CONFIG_PGTABLE_LEVELS <= 3 */ | |
288 | mov pud, pgd | |
289 | b walk_pmds | |
290 | next_pud: | |
291 | b next_pgd | |
292 | .endif | |
293 | ||
294 | /* PMD */ | |
295 | walk_pmds: | |
296 | .if CONFIG_PGTABLE_LEVELS > 2 | |
297 | pte_to_phys cur_pmdp, pud | |
298 | add end_pmdp, cur_pmdp, #(PTRS_PER_PMD * 8) | |
299 | do_pmd: __idmap_kpti_get_pgtable_ent pmd | |
300 | tbnz pmd, #1, walk_ptes | |
301 | __idmap_kpti_put_pgtable_ent_ng pmd | |
302 | next_pmd: | |
303 | add cur_pmdp, cur_pmdp, #8 | |
304 | cmp cur_pmdp, end_pmdp | |
305 | b.ne do_pmd | |
306 | b next_pud | |
307 | .else /* CONFIG_PGTABLE_LEVELS <= 2 */ | |
308 | mov pmd, pud | |
309 | b walk_ptes | |
310 | next_pmd: | |
311 | b next_pud | |
312 | .endif | |
313 | ||
314 | /* PTE */ | |
315 | walk_ptes: | |
316 | pte_to_phys cur_ptep, pmd | |
317 | add end_ptep, cur_ptep, #(PTRS_PER_PTE * 8) | |
318 | do_pte: __idmap_kpti_get_pgtable_ent pte | |
319 | __idmap_kpti_put_pgtable_ent_ng pte | |
320 | next_pte: | |
321 | add cur_ptep, cur_ptep, #8 | |
322 | cmp cur_ptep, end_ptep | |
323 | b.ne do_pte | |
324 | b next_pmd | |
325 | ||
326 | /* Secondary CPUs end up here */ | |
327 | __idmap_kpti_secondary: | |
328 | /* Uninstall swapper before surgery begins */ | |
329 | __idmap_cpu_set_reserved_ttbr1 x18, x17 | |
330 | ||
331 | /* Increment the flag to let the boot CPU we're ready */ | |
332 | 1: ldxr w18, [flag_ptr] | |
333 | add w18, w18, #1 | |
334 | stxr w17, w18, [flag_ptr] | |
335 | cbnz w17, 1b | |
336 | ||
337 | /* Wait for the boot CPU to finish messing around with swapper */ | |
338 | sevl | |
339 | 1: wfe | |
340 | ldxr w18, [flag_ptr] | |
341 | cbnz w18, 1b | |
342 | ||
343 | /* All done, act like nothing happened */ | |
344 | msr ttbr1_el1, swapper_ttb | |
345 | isb | |
346 | ret | |
347 | ||
348 | .unreq cpu | |
349 | .unreq num_cpus | |
350 | .unreq swapper_pa | |
351 | .unreq swapper_ttb | |
352 | .unreq flag_ptr | |
353 | .unreq cur_pgdp | |
354 | .unreq end_pgdp | |
355 | .unreq pgd | |
356 | .unreq cur_pudp | |
357 | .unreq end_pudp | |
358 | .unreq pud | |
359 | .unreq cur_pmdp | |
360 | .unreq end_pmdp | |
361 | .unreq pmd | |
362 | .unreq cur_ptep | |
363 | .unreq end_ptep | |
364 | .unreq pte | |
365 | ENDPROC(idmap_kpti_install_ng_mappings) | |
366 | .popsection | |
367 | #endif | |
368 | ||
9cce7a43 CM |
369 | /* |
370 | * __cpu_setup | |
371 | * | |
372 | * Initialise the processor for turning the MMU on. Return in x0 the | |
373 | * value of the SCTLR_EL1 register. | |
374 | */ | |
b159a095 | 375 | .pushsection ".idmap.text", "awx" |
9cce7a43 | 376 | ENTRY(__cpu_setup) |
fa7aae8a WD |
377 | tlbi vmalle1 // Invalidate local TLB |
378 | dsb nsh | |
9cce7a43 CM |
379 | |
380 | mov x0, #3 << 20 | |
381 | msr cpacr_el1, x0 // Enable FP/ASIMD | |
d8d23fa0 WD |
382 | mov x0, #1 << 12 // Reset mdscr_el1 and disable |
383 | msr mdscr_el1, x0 // access to the DCC from EL0 | |
2ce39ad1 WD |
384 | isb // Unmask debug exceptions now, |
385 | enable_dbg // since this is per-cpu | |
f436b2ac | 386 | reset_pmuserenr_el0 x0 // Disable PMU access from EL0 |
9cce7a43 CM |
387 | /* |
388 | * Memory region attributes for LPAE: | |
389 | * | |
390 | * n = AttrIndx[2:0] | |
391 | * n MAIR | |
392 | * DEVICE_nGnRnE 000 00000000 | |
393 | * DEVICE_nGnRE 001 00000100 | |
394 | * DEVICE_GRE 010 00001100 | |
395 | * NORMAL_NC 011 01000100 | |
396 | * NORMAL 100 11111111 | |
8d446c86 | 397 | * NORMAL_WT 101 10111011 |
9cce7a43 CM |
398 | */ |
399 | ldr x5, =MAIR(0x00, MT_DEVICE_nGnRnE) | \ | |
400 | MAIR(0x04, MT_DEVICE_nGnRE) | \ | |
401 | MAIR(0x0c, MT_DEVICE_GRE) | \ | |
402 | MAIR(0x44, MT_NORMAL_NC) | \ | |
8d446c86 JZZ |
403 | MAIR(0xff, MT_NORMAL) | \ |
404 | MAIR(0xbb, MT_NORMAL_WT) | |
9cce7a43 CM |
405 | msr mair_el1, x5 |
406 | /* | |
407 | * Prepare SCTLR | |
408 | */ | |
409 | adr x5, crval | |
410 | ldp w5, w6, [x5] | |
411 | mrs x0, sctlr_el1 | |
412 | bic x0, x0, x5 // clear bits | |
413 | orr x0, x0, x6 // set bits | |
414 | /* | |
415 | * Set/prepare TCR and TTBR. We use 512GB (39-bit) address range for | |
416 | * both user and kernel. | |
417 | */ | |
35a86976 | 418 | ldr x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \ |
c4f25e12 | 419 | TCR_TG_FLAGS | TCR_ASID16 | TCR_TBI0 | TCR_A1 |
dd006da2 AB |
420 | tcr_set_idmap_t0sz x10, x9 |
421 | ||
87366d8c RMC |
422 | /* |
423 | * Read the PARange bits from ID_AA64MMFR0_EL1 and set the IPS bits in | |
424 | * TCR_EL1. | |
425 | */ | |
426 | mrs x9, ID_AA64MMFR0_EL1 | |
427 | bfi x10, x9, #32, #3 | |
2f4b829c CM |
428 | #ifdef CONFIG_ARM64_HW_AFDBM |
429 | /* | |
430 | * Hardware update of the Access and Dirty bits. | |
431 | */ | |
432 | mrs x9, ID_AA64MMFR1_EL1 | |
433 | and x9, x9, #0xf | |
434 | cbz x9, 2f | |
435 | cmp x9, #2 | |
436 | b.lt 1f | |
437 | orr x10, x10, #TCR_HD // hardware Dirty flag update | |
438 | 1: orr x10, x10, #TCR_HA // hardware Access flag update | |
439 | 2: | |
440 | #endif /* CONFIG_ARM64_HW_AFDBM */ | |
9cce7a43 CM |
441 | msr tcr_el1, x10 |
442 | ret // return to head.S | |
443 | ENDPROC(__cpu_setup) | |
444 | ||
445 | /* | |
9f71ac96 SP |
446 | * We set the desired value explicitly, including those of the |
447 | * reserved bits. The values of bits EE & E0E were set early in | |
448 | * el2_setup, which are left untouched below. | |
449 | * | |
9cce7a43 CM |
450 | * n n T |
451 | * U E WT T UD US IHBS | |
452 | * CE0 XWHW CZ ME TEEA S | |
453 | * .... .IEE .... NEAI TE.I ..AD DEN0 ACAM | |
9f71ac96 SP |
454 | * 0011 0... 1101 ..0. ..0. 10.. .0.. .... < hardware reserved |
455 | * .... .1.. .... 01.1 11.1 ..01 0.01 1101 < software settings | |
9cce7a43 CM |
456 | */ |
457 | .type crval, #object | |
458 | crval: | |
9f71ac96 SP |
459 | .word 0xfcffffff // clear |
460 | .word 0x34d5d91d // set | |
b6113038 | 461 | .popsection |