]>
Commit | Line | Data |
---|---|---|
caab277b | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
9cce7a43 CM |
2 | /* |
3 | * Based on arch/arm/mm/proc.S | |
4 | * | |
5 | * Copyright (C) 2001 Deep Blue Solutions Ltd. | |
6 | * Copyright (C) 2012 ARM Ltd. | |
7 | * Author: Catalin Marinas <catalin.marinas@arm.com> | |
9cce7a43 CM |
8 | */ |
9 | ||
10 | #include <linux/init.h> | |
11 | #include <linux/linkage.h> | |
12 | #include <asm/assembler.h> | |
13 | #include <asm/asm-offsets.h> | |
14 | #include <asm/hwcap.h> | |
9cce7a43 | 15 | #include <asm/pgtable.h> |
cabe1c81 | 16 | #include <asm/pgtable-hwdef.h> |
104a0c02 AP |
17 | #include <asm/cpufeature.h> |
18 | #include <asm/alternative.h> | |
9cce7a43 | 19 | |
35a86976 CM |
20 | #ifdef CONFIG_ARM64_64K_PAGES |
21 | #define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K | |
44eaacf1 SP |
22 | #elif defined(CONFIG_ARM64_16K_PAGES) |
23 | #define TCR_TG_FLAGS TCR_TG0_16K | TCR_TG1_16K | |
24 | #else /* CONFIG_ARM64_4K_PAGES */ | |
35a86976 CM |
25 | #define TCR_TG_FLAGS TCR_TG0_4K | TCR_TG1_4K |
26 | #endif | |
27 | ||
e03e61c3 WD |
28 | #ifdef CONFIG_RANDOMIZE_BASE |
29 | #define TCR_KASLR_FLAGS TCR_NFD1 | |
30 | #else | |
31 | #define TCR_KASLR_FLAGS 0 | |
32 | #endif | |
33 | ||
35a86976 | 34 | #define TCR_SMP_FLAGS TCR_SHARED |
9cce7a43 | 35 | |
35a86976 CM |
36 | /* PTWs cacheable, inner/outer WBWA */ |
37 | #define TCR_CACHE_FLAGS TCR_IRGN_WBWA | TCR_ORGN_WBWA | |
38 | ||
21696c16 AK |
39 | #ifdef CONFIG_KASAN_SW_TAGS |
40 | #define TCR_KASAN_FLAGS TCR_TBI1 | |
41 | #else | |
42 | #define TCR_KASAN_FLAGS 0 | |
43 | #endif | |
44 | ||
9cce7a43 CM |
45 | #define MAIR(attr, mt) ((attr) << ((mt) * 8)) |
46 | ||
af3cfdbf | 47 | #ifdef CONFIG_CPU_PM |
6732bc65 LP |
48 | /** |
49 | * cpu_do_suspend - save CPU registers context | |
50 | * | |
51 | * x0: virtual address of context pointer | |
52 | */ | |
53 | ENTRY(cpu_do_suspend) | |
54 | mrs x2, tpidr_el0 | |
55 | mrs x3, tpidrro_el0 | |
56 | mrs x4, contextidr_el1 | |
827a108e JPB |
57 | mrs x5, osdlr_el1 |
58 | mrs x6, cpacr_el1 | |
59 | mrs x7, tcr_el1 | |
60 | mrs x8, vbar_el1 | |
61 | mrs x9, mdscr_el1 | |
62 | mrs x10, oslsr_el1 | |
63 | mrs x11, sctlr_el1 | |
6d99b689 | 64 | alternative_if_not ARM64_HAS_VIRT_HOST_EXTN |
827a108e | 65 | mrs x12, tpidr_el1 |
6d99b689 | 66 | alternative_else |
827a108e | 67 | mrs x12, tpidr_el2 |
6d99b689 | 68 | alternative_endif |
827a108e | 69 | mrs x13, sp_el0 |
6732bc65 | 70 | stp x2, x3, [x0] |
827a108e JPB |
71 | stp x4, x5, [x0, #16] |
72 | stp x6, x7, [x0, #32] | |
73 | stp x8, x9, [x0, #48] | |
74 | stp x10, x11, [x0, #64] | |
75 | stp x12, x13, [x0, #80] | |
6732bc65 LP |
76 | ret |
77 | ENDPROC(cpu_do_suspend) | |
78 | ||
79 | /** | |
80 | * cpu_do_resume - restore CPU register context | |
81 | * | |
cabe1c81 | 82 | * x0: Address of context pointer |
6732bc65 | 83 | */ |
439e70e2 | 84 | .pushsection ".idmap.text", "awx" |
6732bc65 | 85 | ENTRY(cpu_do_resume) |
6732bc65 LP |
86 | ldp x2, x3, [x0] |
87 | ldp x4, x5, [x0, #16] | |
cabe1c81 JM |
88 | ldp x6, x8, [x0, #32] |
89 | ldp x9, x10, [x0, #48] | |
90 | ldp x11, x12, [x0, #64] | |
623b476f | 91 | ldp x13, x14, [x0, #80] |
6732bc65 LP |
92 | msr tpidr_el0, x2 |
93 | msr tpidrro_el0, x3 | |
94 | msr contextidr_el1, x4 | |
6732bc65 | 95 | msr cpacr_el1, x6 |
cabe1c81 JM |
96 | |
97 | /* Don't change t0sz here, mask those bits when restoring */ | |
827a108e JPB |
98 | mrs x7, tcr_el1 |
99 | bfi x8, x7, TCR_T0SZ_OFFSET, TCR_TxSZ_WIDTH | |
cabe1c81 | 100 | |
6732bc65 LP |
101 | msr tcr_el1, x8 |
102 | msr vbar_el1, x9 | |
744c6c37 JM |
103 | |
104 | /* | |
105 | * __cpu_setup() cleared MDSCR_EL1.MDE and friends, before unmasking | |
106 | * debug exceptions. By restoring MDSCR_EL1 here, we may take a debug | |
0fbeb318 | 107 | * exception. Mask them until local_daif_restore() in cpu_suspend() |
744c6c37 JM |
108 | * resets them. |
109 | */ | |
0fbeb318 | 110 | disable_daif |
6732bc65 | 111 | msr mdscr_el1, x10 |
744c6c37 | 112 | |
cabe1c81 | 113 | msr sctlr_el1, x12 |
6d99b689 | 114 | alternative_if_not ARM64_HAS_VIRT_HOST_EXTN |
623b476f | 115 | msr tpidr_el1, x13 |
6d99b689 JM |
116 | alternative_else |
117 | msr tpidr_el2, x13 | |
118 | alternative_endif | |
623b476f | 119 | msr sp_el0, x14 |
6732bc65 LP |
120 | /* |
121 | * Restore oslsr_el1 by writing oslar_el1 | |
122 | */ | |
827a108e | 123 | msr osdlr_el1, x5 |
6732bc65 LP |
124 | ubfx x11, x11, #1, #1 |
125 | msr oslar_el1, x11 | |
f436b2ac | 126 | reset_pmuserenr_el0 x0 // Disable PMU access from EL0 |
68ddbf09 JM |
127 | |
128 | alternative_if ARM64_HAS_RAS_EXTN | |
129 | msr_s SYS_DISR_EL1, xzr | |
130 | alternative_else_nop_endif | |
131 | ||
6732bc65 LP |
132 | isb |
133 | ret | |
134 | ENDPROC(cpu_do_resume) | |
b6113038 | 135 | .popsection |
6732bc65 LP |
136 | #endif |
137 | ||
9cce7a43 | 138 | /* |
812944e9 | 139 | * cpu_do_switch_mm(pgd_phys, tsk) |
9cce7a43 CM |
140 | * |
141 | * Set the translation table base pointer to be pgd_phys. | |
142 | * | |
143 | * - pgd_phys - physical address of new TTB | |
144 | */ | |
145 | ENTRY(cpu_do_switch_mm) | |
7655abb9 | 146 | mrs x2, ttbr1_el1 |
5aec715d | 147 | mmid x1, x1 // get mm->context.id |
fa0465fc | 148 | phys_to_ttbr x3, x0 |
5ffdfaed VM |
149 | |
150 | alternative_if ARM64_HAS_CNP | |
151 | cbz x1, 1f // skip CNP for reserved ASID | |
152 | orr x3, x3, #TTBR_CNP_BIT | |
153 | 1: | |
154 | alternative_else_nop_endif | |
6b88a32c | 155 | #ifdef CONFIG_ARM64_SW_TTBR0_PAN |
ec89ab50 | 156 | bfi x3, x1, #48, #16 // set the ASID field in TTBR0 |
6b88a32c | 157 | #endif |
7655abb9 WD |
158 | bfi x2, x1, #48, #16 // set the ASID |
159 | msr ttbr1_el1, x2 // in TTBR1 (since TCR.A1 is set) | |
160 | isb | |
ec89ab50 | 161 | msr ttbr0_el1, x3 // now update TTBR0 |
9cce7a43 | 162 | isb |
95e3de35 | 163 | b post_ttbr_update_workaround // Back to C code... |
9cce7a43 CM |
164 | ENDPROC(cpu_do_switch_mm) |
165 | ||
439e70e2 | 166 | .pushsection ".idmap.text", "awx" |
f992b4df WD |
167 | |
168 | .macro __idmap_cpu_set_reserved_ttbr1, tmp1, tmp2 | |
169 | adrp \tmp1, empty_zero_page | |
fa0465fc | 170 | phys_to_ttbr \tmp2, \tmp1 |
e842dfb5 | 171 | offset_ttbr1 \tmp2 |
f992b4df WD |
172 | msr ttbr1_el1, \tmp2 |
173 | isb | |
174 | tlbi vmalle1 | |
175 | dsb nsh | |
176 | isb | |
177 | .endm | |
178 | ||
50e1881d | 179 | /* |
5ffdfaed | 180 | * void idmap_cpu_replace_ttbr1(phys_addr_t ttbr1) |
50e1881d MR |
181 | * |
182 | * This is the low-level counterpart to cpu_replace_ttbr1, and should not be | |
183 | * called by anything else. It can only be executed from a TTBR0 mapping. | |
184 | */ | |
185 | ENTRY(idmap_cpu_replace_ttbr1) | |
0fbeb318 | 186 | save_and_disable_daif flags=x2 |
50e1881d | 187 | |
f992b4df | 188 | __idmap_cpu_set_reserved_ttbr1 x1, x3 |
50e1881d | 189 | |
e842dfb5 | 190 | offset_ttbr1 x0 |
5ffdfaed | 191 | msr ttbr1_el1, x0 |
50e1881d MR |
192 | isb |
193 | ||
0fbeb318 | 194 | restore_daif x2 |
50e1881d MR |
195 | |
196 | ret | |
197 | ENDPROC(idmap_cpu_replace_ttbr1) | |
198 | .popsection | |
199 | ||
f992b4df | 200 | #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 |
439e70e2 | 201 | .pushsection ".idmap.text", "awx" |
f992b4df WD |
202 | |
203 | .macro __idmap_kpti_get_pgtable_ent, type | |
204 | dc cvac, cur_\()\type\()p // Ensure any existing dirty | |
205 | dmb sy // lines are written back before | |
206 | ldr \type, [cur_\()\type\()p] // loading the entry | |
2ce77f6d WD |
207 | tbz \type, #0, skip_\()\type // Skip invalid and |
208 | tbnz \type, #11, skip_\()\type // non-global entries | |
f992b4df WD |
209 | .endm |
210 | ||
211 | .macro __idmap_kpti_put_pgtable_ent_ng, type | |
212 | orr \type, \type, #PTE_NG // Same bit for blocks and pages | |
71c8fc0c WD |
213 | str \type, [cur_\()\type\()p] // Update the entry and ensure |
214 | dmb sy // that it is visible to all | |
215 | dc civac, cur_\()\type\()p // CPUs. | |
f992b4df WD |
216 | .endm |
217 | ||
218 | /* | |
219 | * void __kpti_install_ng_mappings(int cpu, int num_cpus, phys_addr_t swapper) | |
220 | * | |
221 | * Called exactly once from stop_machine context by each CPU found during boot. | |
222 | */ | |
223 | __idmap_kpti_flag: | |
224 | .long 1 | |
225 | ENTRY(idmap_kpti_install_ng_mappings) | |
226 | cpu .req w0 | |
227 | num_cpus .req w1 | |
228 | swapper_pa .req x2 | |
229 | swapper_ttb .req x3 | |
230 | flag_ptr .req x4 | |
231 | cur_pgdp .req x5 | |
232 | end_pgdp .req x6 | |
233 | pgd .req x7 | |
234 | cur_pudp .req x8 | |
235 | end_pudp .req x9 | |
236 | pud .req x10 | |
237 | cur_pmdp .req x11 | |
238 | end_pmdp .req x12 | |
239 | pmd .req x13 | |
240 | cur_ptep .req x14 | |
241 | end_ptep .req x15 | |
242 | pte .req x16 | |
243 | ||
244 | mrs swapper_ttb, ttbr1_el1 | |
e842dfb5 | 245 | restore_ttbr1 swapper_ttb |
f992b4df WD |
246 | adr flag_ptr, __idmap_kpti_flag |
247 | ||
248 | cbnz cpu, __idmap_kpti_secondary | |
249 | ||
250 | /* We're the boot CPU. Wait for the others to catch up */ | |
251 | sevl | |
252 | 1: wfe | |
253 | ldaxr w18, [flag_ptr] | |
254 | eor w18, w18, num_cpus | |
255 | cbnz w18, 1b | |
256 | ||
257 | /* We need to walk swapper, so turn off the MMU. */ | |
258 | pre_disable_mmu_workaround | |
259 | mrs x18, sctlr_el1 | |
260 | bic x18, x18, #SCTLR_ELx_M | |
261 | msr sctlr_el1, x18 | |
262 | isb | |
263 | ||
264 | /* Everybody is enjoying the idmap, so we can rewrite swapper. */ | |
265 | /* PGD */ | |
266 | mov cur_pgdp, swapper_pa | |
267 | add end_pgdp, cur_pgdp, #(PTRS_PER_PGD * 8) | |
268 | do_pgd: __idmap_kpti_get_pgtable_ent pgd | |
269 | tbnz pgd, #1, walk_puds | |
f992b4df | 270 | next_pgd: |
2ce77f6d WD |
271 | __idmap_kpti_put_pgtable_ent_ng pgd |
272 | skip_pgd: | |
f992b4df WD |
273 | add cur_pgdp, cur_pgdp, #8 |
274 | cmp cur_pgdp, end_pgdp | |
275 | b.ne do_pgd | |
276 | ||
277 | /* Publish the updated tables and nuke all the TLBs */ | |
278 | dsb sy | |
279 | tlbi vmalle1is | |
280 | dsb ish | |
281 | isb | |
282 | ||
283 | /* We're done: fire up the MMU again */ | |
284 | mrs x18, sctlr_el1 | |
285 | orr x18, x18, #SCTLR_ELx_M | |
286 | msr sctlr_el1, x18 | |
287 | isb | |
288 | ||
289 | /* Set the flag to zero to indicate that we're all done */ | |
290 | str wzr, [flag_ptr] | |
291 | ret | |
292 | ||
293 | /* PUD */ | |
294 | walk_puds: | |
295 | .if CONFIG_PGTABLE_LEVELS > 3 | |
296 | pte_to_phys cur_pudp, pgd | |
297 | add end_pudp, cur_pudp, #(PTRS_PER_PUD * 8) | |
298 | do_pud: __idmap_kpti_get_pgtable_ent pud | |
299 | tbnz pud, #1, walk_pmds | |
f992b4df | 300 | next_pud: |
2ce77f6d WD |
301 | __idmap_kpti_put_pgtable_ent_ng pud |
302 | skip_pud: | |
f992b4df WD |
303 | add cur_pudp, cur_pudp, 8 |
304 | cmp cur_pudp, end_pudp | |
305 | b.ne do_pud | |
306 | b next_pgd | |
307 | .else /* CONFIG_PGTABLE_LEVELS <= 3 */ | |
308 | mov pud, pgd | |
309 | b walk_pmds | |
310 | next_pud: | |
311 | b next_pgd | |
312 | .endif | |
313 | ||
314 | /* PMD */ | |
315 | walk_pmds: | |
316 | .if CONFIG_PGTABLE_LEVELS > 2 | |
317 | pte_to_phys cur_pmdp, pud | |
318 | add end_pmdp, cur_pmdp, #(PTRS_PER_PMD * 8) | |
319 | do_pmd: __idmap_kpti_get_pgtable_ent pmd | |
320 | tbnz pmd, #1, walk_ptes | |
f992b4df | 321 | next_pmd: |
2ce77f6d WD |
322 | __idmap_kpti_put_pgtable_ent_ng pmd |
323 | skip_pmd: | |
f992b4df WD |
324 | add cur_pmdp, cur_pmdp, #8 |
325 | cmp cur_pmdp, end_pmdp | |
326 | b.ne do_pmd | |
327 | b next_pud | |
328 | .else /* CONFIG_PGTABLE_LEVELS <= 2 */ | |
329 | mov pmd, pud | |
330 | b walk_ptes | |
331 | next_pmd: | |
332 | b next_pud | |
333 | .endif | |
334 | ||
335 | /* PTE */ | |
336 | walk_ptes: | |
337 | pte_to_phys cur_ptep, pmd | |
338 | add end_ptep, cur_ptep, #(PTRS_PER_PTE * 8) | |
339 | do_pte: __idmap_kpti_get_pgtable_ent pte | |
340 | __idmap_kpti_put_pgtable_ent_ng pte | |
2ce77f6d | 341 | skip_pte: |
f992b4df WD |
342 | add cur_ptep, cur_ptep, #8 |
343 | cmp cur_ptep, end_ptep | |
344 | b.ne do_pte | |
345 | b next_pmd | |
346 | ||
347 | /* Secondary CPUs end up here */ | |
348 | __idmap_kpti_secondary: | |
349 | /* Uninstall swapper before surgery begins */ | |
350 | __idmap_cpu_set_reserved_ttbr1 x18, x17 | |
351 | ||
352 | /* Increment the flag to let the boot CPU we're ready */ | |
353 | 1: ldxr w18, [flag_ptr] | |
354 | add w18, w18, #1 | |
355 | stxr w17, w18, [flag_ptr] | |
356 | cbnz w17, 1b | |
357 | ||
358 | /* Wait for the boot CPU to finish messing around with swapper */ | |
359 | sevl | |
360 | 1: wfe | |
361 | ldxr w18, [flag_ptr] | |
362 | cbnz w18, 1b | |
363 | ||
364 | /* All done, act like nothing happened */ | |
e842dfb5 | 365 | offset_ttbr1 swapper_ttb |
f992b4df WD |
366 | msr ttbr1_el1, swapper_ttb |
367 | isb | |
368 | ret | |
369 | ||
370 | .unreq cpu | |
371 | .unreq num_cpus | |
372 | .unreq swapper_pa | |
373 | .unreq swapper_ttb | |
374 | .unreq flag_ptr | |
375 | .unreq cur_pgdp | |
376 | .unreq end_pgdp | |
377 | .unreq pgd | |
378 | .unreq cur_pudp | |
379 | .unreq end_pudp | |
380 | .unreq pud | |
381 | .unreq cur_pmdp | |
382 | .unreq end_pmdp | |
383 | .unreq pmd | |
384 | .unreq cur_ptep | |
385 | .unreq end_ptep | |
386 | .unreq pte | |
387 | ENDPROC(idmap_kpti_install_ng_mappings) | |
388 | .popsection | |
389 | #endif | |
390 | ||
9cce7a43 CM |
391 | /* |
392 | * __cpu_setup | |
393 | * | |
394 | * Initialise the processor for turning the MMU on. Return in x0 the | |
395 | * value of the SCTLR_EL1 register. | |
396 | */ | |
439e70e2 | 397 | .pushsection ".idmap.text", "awx" |
9cce7a43 | 398 | ENTRY(__cpu_setup) |
fa7aae8a WD |
399 | tlbi vmalle1 // Invalidate local TLB |
400 | dsb nsh | |
9cce7a43 CM |
401 | |
402 | mov x0, #3 << 20 | |
403 | msr cpacr_el1, x0 // Enable FP/ASIMD | |
d8d23fa0 WD |
404 | mov x0, #1 << 12 // Reset mdscr_el1 and disable |
405 | msr mdscr_el1, x0 // access to the DCC from EL0 | |
2ce39ad1 WD |
406 | isb // Unmask debug exceptions now, |
407 | enable_dbg // since this is per-cpu | |
f436b2ac | 408 | reset_pmuserenr_el0 x0 // Disable PMU access from EL0 |
9cce7a43 CM |
409 | /* |
410 | * Memory region attributes for LPAE: | |
411 | * | |
412 | * n = AttrIndx[2:0] | |
413 | * n MAIR | |
414 | * DEVICE_nGnRnE 000 00000000 | |
415 | * DEVICE_nGnRE 001 00000100 | |
416 | * DEVICE_GRE 010 00001100 | |
417 | * NORMAL_NC 011 01000100 | |
418 | * NORMAL 100 11111111 | |
8d446c86 | 419 | * NORMAL_WT 101 10111011 |
9cce7a43 CM |
420 | */ |
421 | ldr x5, =MAIR(0x00, MT_DEVICE_nGnRnE) | \ | |
422 | MAIR(0x04, MT_DEVICE_nGnRE) | \ | |
423 | MAIR(0x0c, MT_DEVICE_GRE) | \ | |
424 | MAIR(0x44, MT_NORMAL_NC) | \ | |
8d446c86 JZZ |
425 | MAIR(0xff, MT_NORMAL) | \ |
426 | MAIR(0xbb, MT_NORMAL_WT) | |
9cce7a43 CM |
427 | msr mair_el1, x5 |
428 | /* | |
429 | * Prepare SCTLR | |
430 | */ | |
7a00d68e | 431 | mov_q x0, SCTLR_EL1_SET |
9cce7a43 CM |
432 | /* |
433 | * Set/prepare TCR and TTBR. We use 512GB (39-bit) address range for | |
434 | * both user and kernel. | |
435 | */ | |
35a86976 | 436 | ldr x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \ |
e03e61c3 | 437 | TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_ASID16 | \ |
21696c16 | 438 | TCR_TBI0 | TCR_A1 | TCR_KASAN_FLAGS |
3e32131a | 439 | tcr_clear_errata_bits x10, x9, x5 |
67e7fdfc | 440 | |
68d23da4 WD |
441 | #ifdef CONFIG_ARM64_USER_VA_BITS_52 |
442 | ldr_l x9, vabits_user | |
67e7fdfc SC |
443 | sub x9, xzr, x9 |
444 | add x9, x9, #64 | |
445 | #else | |
446 | ldr_l x9, idmap_t0sz | |
447 | #endif | |
448 | tcr_set_t0sz x10, x9 | |
dd006da2 | 449 | |
87366d8c | 450 | /* |
787fd1d0 | 451 | * Set the IPS bits in TCR_EL1. |
87366d8c | 452 | */ |
787fd1d0 | 453 | tcr_compute_pa_size x10, #TCR_IPS_SHIFT, x5, x6 |
2f4b829c CM |
454 | #ifdef CONFIG_ARM64_HW_AFDBM |
455 | /* | |
05abb595 SP |
456 | * Enable hardware update of the Access Flags bit. |
457 | * Hardware dirty bit management is enabled later, | |
458 | * via capabilities. | |
2f4b829c CM |
459 | */ |
460 | mrs x9, ID_AA64MMFR1_EL1 | |
461 | and x9, x9, #0xf | |
05abb595 SP |
462 | cbz x9, 1f |
463 | orr x10, x10, #TCR_HA // hardware Access flag update | |
464 | 1: | |
2f4b829c | 465 | #endif /* CONFIG_ARM64_HW_AFDBM */ |
9cce7a43 CM |
466 | msr tcr_el1, x10 |
467 | ret // return to head.S | |
468 | ENDPROC(__cpu_setup) |