]>
Commit | Line | Data |
---|---|---|
caab277b | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
9703d9d7 CM |
2 | /* |
3 | * Low-level CPU initialisation | |
4 | * Based on arch/arm/kernel/head.S | |
5 | * | |
6 | * Copyright (C) 1994-2002 Russell King | |
7 | * Copyright (C) 2003-2012 ARM Ltd. | |
8 | * Authors: Catalin Marinas <catalin.marinas@arm.com> | |
9 | * Will Deacon <will.deacon@arm.com> | |
9703d9d7 CM |
10 | */ |
11 | ||
12 | #include <linux/linkage.h> | |
13 | #include <linux/init.h> | |
65fddcfc | 14 | #include <linux/pgtable.h> |
9703d9d7 | 15 | |
62a679cb | 16 | #include <asm/asm_pointer_auth.h> |
9703d9d7 | 17 | #include <asm/assembler.h> |
08cdac61 | 18 | #include <asm/boot.h> |
7d7b720a | 19 | #include <asm/bug.h> |
9703d9d7 CM |
20 | #include <asm/ptrace.h> |
21 | #include <asm/asm-offsets.h> | |
c218bca7 | 22 | #include <asm/cache.h> |
0359b0e2 | 23 | #include <asm/cputype.h> |
78869f0f | 24 | #include <asm/el2_setup.h> |
1e48ef7f | 25 | #include <asm/elf.h> |
f56063c5 | 26 | #include <asm/image.h> |
87d1587b | 27 | #include <asm/kernel-pgtable.h> |
1f364c8c | 28 | #include <asm/kvm_arm.h> |
9703d9d7 | 29 | #include <asm/memory.h> |
9703d9d7 | 30 | #include <asm/pgtable-hwdef.h> |
9703d9d7 | 31 | #include <asm/page.h> |
5287569a | 32 | #include <asm/scs.h> |
bb905274 | 33 | #include <asm/smp.h> |
4bf8b96e SP |
34 | #include <asm/sysreg.h> |
35 | #include <asm/thread_info.h> | |
f35a9205 | 36 | #include <asm/virt.h> |
9703d9d7 | 37 | |
b5f4a214 AB |
38 | #include "efi-header.S" |
39 | ||
120dc60d | 40 | #define __PHYS_OFFSET KERNEL_START |
9703d9d7 | 41 | |
120dc60d | 42 | #if (PAGE_OFFSET & 0x1fffff) != 0 |
da57a369 | 43 | #error PAGE_OFFSET must be at least 2MB aligned |
9703d9d7 CM |
44 | #endif |
45 | ||
9703d9d7 CM |
46 | /* |
47 | * Kernel startup entry point. | |
48 | * --------------------------- | |
49 | * | |
50 | * The requirements are: | |
51 | * MMU = off, D-cache = off, I-cache = on or off, | |
52 | * x0 = physical address to the FDT blob. | |
53 | * | |
54 | * This code is mostly position independent so you call this at | |
120dc60d | 55 | * __pa(PAGE_OFFSET). |
9703d9d7 CM |
56 | * |
57 | * Note that the callee-saved registers are used for storing variables | |
58 | * that are useful before the MMU is enabled. The allocations are described | |
59 | * in the entry routines. | |
60 | */ | |
61 | __HEAD | |
9703d9d7 CM |
62 | /* |
63 | * DO NOT MODIFY. Image header expected by Linux boot-loaders. | |
64 | */ | |
7919385b | 65 | efi_signature_nop // special NOP to identity as PE/COFF executable |
348a625d | 66 | b primary_entry // branch to kernel start, magic |
120dc60d | 67 | .quad 0 // Image load offset from start of RAM, little-endian |
6ad1fe5d AB |
68 | le64sym _kernel_size_le // Effective size of kernel image, little-endian |
69 | le64sym _kernel_flags_le // Informative flags, little-endian | |
4370eec0 RF |
70 | .quad 0 // reserved |
71 | .quad 0 // reserved | |
72 | .quad 0 // reserved | |
f56063c5 | 73 | .ascii ARM64_IMAGE_MAGIC // Magic number |
7919385b | 74 | .long .Lpe_header_offset // Offset to the PE header. |
3c7f2550 | 75 | |
b5f4a214 | 76 | __EFI_PE_HEADER |
9703d9d7 | 77 | |
546c8c44 AB |
78 | __INIT |
79 | ||
a9be2ee0 AB |
80 | /* |
81 | * The following callee saved general purpose registers are used on the | |
82 | * primary lowlevel boot path: | |
83 | * | |
84 | * Register Scope Purpose | |
348a625d AB |
85 | * x21 primary_entry() .. start_kernel() FDT pointer passed at boot in x0 |
86 | * x23 primary_entry() .. start_kernel() physical misalignment/KASLR offset | |
87 | * x28 __create_page_tables() callee preserved temp register | |
88 | * x19/x20 __primary_switch() callee preserved temp registers | |
89 | * x24 __primary_switch() .. relocate_kernel() current RELR displacement | |
a9be2ee0 | 90 | */ |
348a625d | 91 | SYM_CODE_START(primary_entry) |
da9c177d | 92 | bl preserve_boot_args |
ecbb11ab | 93 | bl init_kernel_el // w0=cpu_boot_mode |
b929fe32 AB |
94 | adrp x23, __PHYS_OFFSET |
95 | and x23, x23, MIN_KIMG_ALIGN - 1 // KASLR offset, defaults to 0 | |
828e9834 | 96 | bl set_cpu_boot_mode_flag |
aea73abb | 97 | bl __create_page_tables |
9703d9d7 | 98 | /* |
a591ede4 MZ |
99 | * The following calls CPU setup code, see arch/arm64/mm/proc.S for |
100 | * details. | |
9703d9d7 CM |
101 | * On return, the CPU will be ready for the MMU to be turned on and |
102 | * the TCR will have been set. | |
103 | */ | |
0cd3defe | 104 | bl __cpu_setup // initialise processor |
3c5e9f23 | 105 | b __primary_switch |
348a625d | 106 | SYM_CODE_END(primary_entry) |
9703d9d7 | 107 | |
da9c177d AB |
108 | /* |
109 | * Preserve the arguments passed by the bootloader in x0 .. x3 | |
110 | */ | |
ebdf44a1 | 111 | SYM_CODE_START_LOCAL(preserve_boot_args) |
da9c177d AB |
112 | mov x21, x0 // x21=FDT |
113 | ||
114 | adr_l x0, boot_args // record the contents of | |
115 | stp x21, x1, [x0] // x0 .. x3 at kernel entry | |
116 | stp x2, x3, [x0, #16] | |
117 | ||
118 | dmb sy // needed before dc ivac with | |
119 | // MMU off | |
120 | ||
e3974adb | 121 | add x1, x0, #0x20 // 4 x 8 bytes |
fade9c2c | 122 | b dcache_inval_poc // tail call |
ebdf44a1 | 123 | SYM_CODE_END(preserve_boot_args) |
da9c177d | 124 | |
034edabe LA |
125 | /* |
126 | * Macro to create a table entry to the next page. | |
127 | * | |
128 | * tbl: page table address | |
129 | * virt: virtual address | |
130 | * shift: #imm page table shift | |
131 | * ptrs: #imm pointers per table page | |
132 | * | |
133 | * Preserves: virt | |
fa2a8445 | 134 | * Corrupts: ptrs, tmp1, tmp2 |
034edabe LA |
135 | * Returns: tbl -> next level table page address |
136 | */ | |
137 | .macro create_table_entry, tbl, virt, shift, ptrs, tmp1, tmp2 | |
e6d588a8 | 138 | add \tmp1, \tbl, #PAGE_SIZE |
79ddab3b | 139 | phys_to_pte \tmp2, \tmp1 |
e6d588a8 | 140 | orr \tmp2, \tmp2, #PMD_TYPE_TABLE // address of next table and entry type |
034edabe | 141 | lsr \tmp1, \virt, #\shift |
fa2a8445 KM |
142 | sub \ptrs, \ptrs, #1 |
143 | and \tmp1, \tmp1, \ptrs // table index | |
034edabe LA |
144 | str \tmp2, [\tbl, \tmp1, lsl #3] |
145 | add \tbl, \tbl, #PAGE_SIZE // next level table page | |
146 | .endm | |
147 | ||
148 | /* | |
0370b31e SC |
149 | * Macro to populate page table entries, these entries can be pointers to the next level |
150 | * or last level entries pointing to physical memory. | |
034edabe | 151 | * |
0370b31e SC |
152 | * tbl: page table address |
153 | * rtbl: pointer to page table or physical memory | |
154 | * index: start index to write | |
155 | * eindex: end index to write - [index, eindex] written to | |
156 | * flags: flags for pagetable entry to or in | |
157 | * inc: increment to rtbl between each entry | |
158 | * tmp1: temporary variable | |
159 | * | |
160 | * Preserves: tbl, eindex, flags, inc | |
161 | * Corrupts: index, tmp1 | |
162 | * Returns: rtbl | |
034edabe | 163 | */ |
0370b31e | 164 | .macro populate_entries, tbl, rtbl, index, eindex, flags, inc, tmp1 |
79ddab3b | 165 | .Lpe\@: phys_to_pte \tmp1, \rtbl |
0370b31e SC |
166 | orr \tmp1, \tmp1, \flags // tmp1 = table entry |
167 | str \tmp1, [\tbl, \index, lsl #3] | |
168 | add \rtbl, \rtbl, \inc // rtbl = pa next level | |
169 | add \index, \index, #1 | |
170 | cmp \index, \eindex | |
171 | b.ls .Lpe\@ | |
172 | .endm | |
173 | ||
174 | /* | |
175 | * Compute indices of table entries from virtual address range. If multiple entries | |
176 | * were needed in the previous page table level then the next page table level is assumed | |
177 | * to be composed of multiple pages. (This effectively scales the end index). | |
178 | * | |
179 | * vstart: virtual address of start of range | |
90268574 | 180 | * vend: virtual address of end of range - we map [vstart, vend] |
0370b31e SC |
181 | * shift: shift used to transform virtual address into index |
182 | * ptrs: number of entries in page table | |
183 | * istart: index in table corresponding to vstart | |
184 | * iend: index in table corresponding to vend | |
185 | * count: On entry: how many extra entries were required in previous level, scales | |
186 | * our end index. | |
187 | * On exit: returns how many extra entries required for next page table level | |
188 | * | |
189 | * Preserves: vstart, vend, shift, ptrs | |
190 | * Returns: istart, iend, count | |
191 | */ | |
192 | .macro compute_indices, vstart, vend, shift, ptrs, istart, iend, count | |
193 | lsr \iend, \vend, \shift | |
194 | mov \istart, \ptrs | |
195 | sub \istart, \istart, #1 | |
196 | and \iend, \iend, \istart // iend = (vend >> shift) & (ptrs - 1) | |
197 | mov \istart, \ptrs | |
198 | mul \istart, \istart, \count | |
c70fe14f | 199 | add \iend, \iend, \istart // iend += count * ptrs |
0370b31e SC |
200 | // our entries span multiple tables |
201 | ||
202 | lsr \istart, \vstart, \shift | |
203 | mov \count, \ptrs | |
204 | sub \count, \count, #1 | |
205 | and \istart, \istart, \count | |
206 | ||
207 | sub \count, \iend, \istart | |
034edabe LA |
208 | .endm |
209 | ||
210 | /* | |
0370b31e SC |
211 | * Map memory for specified virtual address range. Each level of page table needed supports |
212 | * multiple entries. If a level requires n entries the next page table level is assumed to be | |
213 | * formed from n pages. | |
214 | * | |
215 | * tbl: location of page table | |
216 | * rtbl: address to be used for first level page table entry (typically tbl + PAGE_SIZE) | |
90268574 MR |
217 | * vstart: virtual address of start of range |
218 | * vend: virtual address of end of range - we map [vstart, vend - 1] | |
0370b31e SC |
219 | * flags: flags to use to map last level entries |
220 | * phys: physical address corresponding to vstart - physical memory is contiguous | |
221 | * pgds: the number of pgd entries | |
034edabe | 222 | * |
0370b31e | 223 | * Temporaries: istart, iend, tmp, count, sv - these need to be different registers |
90268574 MR |
224 | * Preserves: vstart, flags |
225 | * Corrupts: tbl, rtbl, vend, istart, iend, tmp, count, sv | |
034edabe | 226 | */ |
0370b31e | 227 | .macro map_memory, tbl, rtbl, vstart, vend, flags, phys, pgds, istart, iend, tmp, count, sv |
90268574 | 228 | sub \vend, \vend, #1 |
0370b31e SC |
229 | add \rtbl, \tbl, #PAGE_SIZE |
230 | mov \sv, \rtbl | |
231 | mov \count, #0 | |
232 | compute_indices \vstart, \vend, #PGDIR_SHIFT, \pgds, \istart, \iend, \count | |
233 | populate_entries \tbl, \rtbl, \istart, \iend, #PMD_TYPE_TABLE, #PAGE_SIZE, \tmp | |
234 | mov \tbl, \sv | |
235 | mov \sv, \rtbl | |
236 | ||
237 | #if SWAPPER_PGTABLE_LEVELS > 3 | |
238 | compute_indices \vstart, \vend, #PUD_SHIFT, #PTRS_PER_PUD, \istart, \iend, \count | |
239 | populate_entries \tbl, \rtbl, \istart, \iend, #PMD_TYPE_TABLE, #PAGE_SIZE, \tmp | |
240 | mov \tbl, \sv | |
241 | mov \sv, \rtbl | |
242 | #endif | |
243 | ||
244 | #if SWAPPER_PGTABLE_LEVELS > 2 | |
245 | compute_indices \vstart, \vend, #SWAPPER_TABLE_SHIFT, #PTRS_PER_PMD, \istart, \iend, \count | |
246 | populate_entries \tbl, \rtbl, \istart, \iend, #PMD_TYPE_TABLE, #PAGE_SIZE, \tmp | |
247 | mov \tbl, \sv | |
248 | #endif | |
249 | ||
250 | compute_indices \vstart, \vend, #SWAPPER_BLOCK_SHIFT, #PTRS_PER_PTE, \istart, \iend, \count | |
251 | bic \count, \phys, #SWAPPER_BLOCK_SIZE - 1 | |
252 | populate_entries \tbl, \count, \istart, \iend, \flags, #SWAPPER_BLOCK_SIZE, \tmp | |
034edabe LA |
253 | .endm |
254 | ||
255 | /* | |
256 | * Setup the initial page tables. We only setup the barest amount which is | |
257 | * required to get the kernel running. The following sections are required: | |
258 | * - identity mapping to enable the MMU (low address, TTBR0) | |
259 | * - first few MB of the kernel linear mapping to jump to once the MMU has | |
61bd93ce | 260 | * been enabled |
034edabe | 261 | */ |
c63d9f82 | 262 | SYM_FUNC_START_LOCAL(__create_page_tables) |
f80fb3a3 | 263 | mov x28, lr |
034edabe LA |
264 | |
265 | /* | |
8eb7e28d JY |
266 | * Invalidate the init page tables to avoid potential dirty cache lines |
267 | * being evicted. Other page tables are allocated in rodata as part of | |
268 | * the kernel image, and thus are clean to the PoC per the boot | |
269 | * protocol. | |
034edabe | 270 | */ |
8eb7e28d | 271 | adrp x0, init_pg_dir |
2b5548b6 | 272 | adrp x1, init_pg_end |
fade9c2c | 273 | bl dcache_inval_poc |
034edabe LA |
274 | |
275 | /* | |
8eb7e28d | 276 | * Clear the init page tables. |
034edabe | 277 | */ |
8eb7e28d | 278 | adrp x0, init_pg_dir |
2b5548b6 | 279 | adrp x1, init_pg_end |
0370b31e | 280 | sub x1, x1, x0 |
034edabe LA |
281 | 1: stp xzr, xzr, [x0], #16 |
282 | stp xzr, xzr, [x0], #16 | |
283 | stp xzr, xzr, [x0], #16 | |
284 | stp xzr, xzr, [x0], #16 | |
d46befef RM |
285 | subs x1, x1, #64 |
286 | b.ne 1b | |
034edabe | 287 | |
0629c791 | 288 | mov_q x7, SWAPPER_MM_MMUFLAGS |
034edabe LA |
289 | |
290 | /* | |
291 | * Create the identity mapping. | |
292 | */ | |
aea73abb | 293 | adrp x0, idmap_pg_dir |
5dfe9d7d | 294 | adrp x3, __idmap_text_start // __pa(__idmap_text_start) |
dd006da2 | 295 | |
b6d00d47 | 296 | #ifdef CONFIG_ARM64_VA_BITS_52 |
67e7fdfc SC |
297 | mrs_s x6, SYS_ID_AA64MMFR2_EL1 |
298 | and x6, x6, #(0xf << ID_AA64MMFR2_LVA_SHIFT) | |
299 | mov x5, #52 | |
300 | cbnz x6, 1f | |
301 | #endif | |
90ec95cd | 302 | mov x5, #VA_BITS_MIN |
67e7fdfc | 303 | 1: |
5383cc6e | 304 | adr_l x6, vabits_actual |
67e7fdfc SC |
305 | str x5, [x6] |
306 | dmb sy | |
307 | dc ivac, x6 // Invalidate potentially stale cache line | |
308 | ||
dd006da2 | 309 | /* |
fa2a8445 KM |
310 | * VA_BITS may be too small to allow for an ID mapping to be created |
311 | * that covers system RAM if that is located sufficiently high in the | |
312 | * physical address space. So for the ID map, use an extended virtual | |
313 | * range in that case, and configure an additional translation level | |
314 | * if needed. | |
315 | * | |
dd006da2 | 316 | * Calculate the maximum allowed value for TCR_EL1.T0SZ so that the |
5dfe9d7d | 317 | * entire ID map region can be mapped. As T0SZ == (64 - #bits used), |
dd006da2 | 318 | * this number conveniently equals the number of leading zeroes in |
5dfe9d7d | 319 | * the physical address of __idmap_text_end. |
dd006da2 | 320 | */ |
5dfe9d7d | 321 | adrp x5, __idmap_text_end |
dd006da2 | 322 | clz x5, x5 |
7ba8f2b2 | 323 | cmp x5, TCR_T0SZ(VA_BITS_MIN) // default T0SZ small enough? |
fa2a8445 | 324 | b.ge 1f // .. then skip VA range extension |
dd006da2 | 325 | |
0c20856c MR |
326 | adr_l x6, idmap_t0sz |
327 | str x5, [x6] | |
328 | dmb sy | |
329 | dc ivac, x6 // Invalidate potentially stale cache line | |
dd006da2 | 330 | |
fa2a8445 KM |
331 | #if (VA_BITS < 48) |
332 | #define EXTRA_SHIFT (PGDIR_SHIFT + PAGE_SHIFT - 3) | |
333 | #define EXTRA_PTRS (1 << (PHYS_MASK_SHIFT - EXTRA_SHIFT)) | |
334 | ||
335 | /* | |
336 | * If VA_BITS < 48, we have to configure an additional table level. | |
337 | * First, we have to verify our assumption that the current value of | |
338 | * VA_BITS was chosen such that all translation levels are fully | |
339 | * utilised, and that lowering T0SZ will always result in an additional | |
340 | * translation level to be configured. | |
341 | */ | |
342 | #if VA_BITS != EXTRA_SHIFT | |
343 | #error "Mismatch between VA_BITS and page size/number of translation levels" | |
dd006da2 AB |
344 | #endif |
345 | ||
fa2a8445 KM |
346 | mov x4, EXTRA_PTRS |
347 | create_table_entry x0, x3, EXTRA_SHIFT, x4, x5, x6 | |
348 | #else | |
349 | /* | |
350 | * If VA_BITS == 48, we don't have to configure an additional | |
351 | * translation level, but the top-level table has more entries. | |
352 | */ | |
353 | mov x4, #1 << (PHYS_MASK_SHIFT - PGDIR_SHIFT) | |
354 | str_l x4, idmap_ptrs_per_pgd, x5 | |
355 | #endif | |
356 | 1: | |
357 | ldr_l x4, idmap_ptrs_per_pgd | |
5dfe9d7d | 358 | adr_l x6, __idmap_text_end // __pa(__idmap_text_end) |
0370b31e SC |
359 | |
360 | map_memory x0, x1, x3, x6, x7, x3, x4, x10, x11, x12, x13, x14 | |
034edabe LA |
361 | |
362 | /* | |
363 | * Map the kernel image (starting with PHYS_OFFSET). | |
364 | */ | |
2b5548b6 | 365 | adrp x0, init_pg_dir |
120dc60d | 366 | mov_q x5, KIMAGE_VADDR // compile time __va(_text) |
f80fb3a3 | 367 | add x5, x5, x23 // add KASLR displacement |
fa2a8445 | 368 | mov x4, PTRS_PER_PGD |
18b9c0d6 AB |
369 | adrp x6, _end // runtime __pa(_end) |
370 | adrp x3, _text // runtime __pa(_text) | |
371 | sub x6, x6, x3 // _end - _text | |
372 | add x6, x6, x5 // runtime __va(_end) | |
0370b31e SC |
373 | |
374 | map_memory x0, x1, x5, x6, x7, x3, x4, x10, x11, x12, x13, x14 | |
034edabe | 375 | |
034edabe LA |
376 | /* |
377 | * Since the page tables have been populated with non-cacheable | |
9d2d75ed GS |
378 | * accesses (MMU disabled), invalidate those tables again to |
379 | * remove any speculatively loaded cache lines. | |
034edabe | 380 | */ |
9d2d75ed GS |
381 | dmb sy |
382 | ||
aea73abb | 383 | adrp x0, idmap_pg_dir |
9d2d75ed | 384 | adrp x1, idmap_pg_end |
fade9c2c | 385 | bl dcache_inval_poc |
9d2d75ed GS |
386 | |
387 | adrp x0, init_pg_dir | |
2b5548b6 | 388 | adrp x1, init_pg_end |
fade9c2c | 389 | bl dcache_inval_poc |
034edabe | 390 | |
f80fb3a3 | 391 | ret x28 |
c63d9f82 | 392 | SYM_FUNC_END(__create_page_tables) |
034edabe | 393 | |
7d7b720a | 394 | /* |
8e334d72 MR |
395 | * Initialize CPU registers with task-specific and cpu-specific context. |
396 | * | |
7d7b720a MV |
397 | * Create a final frame record at task_pt_regs(current)->stackframe, so |
398 | * that the unwinder can identify the final frame record of any task by | |
399 | * its location in the task stack. We reserve the entire pt_regs space | |
400 | * for consistency with user tasks and kthreads. | |
401 | */ | |
3d8c1a01 | 402 | .macro init_cpu_task tsk, tmp1, tmp2 |
8e334d72 MR |
403 | msr sp_el0, \tsk |
404 | ||
3d8c1a01 MR |
405 | ldr \tmp1, [\tsk, #TSK_STACK] |
406 | add sp, \tmp1, #THREAD_SIZE | |
7d7b720a | 407 | sub sp, sp, #PT_REGS_SIZE |
8e334d72 | 408 | |
7d7b720a MV |
409 | stp xzr, xzr, [sp, #S_STACKFRAME] |
410 | add x29, sp, #S_STACKFRAME | |
8e334d72 | 411 | |
16c230b3 | 412 | scs_load \tsk |
3d8c1a01 MR |
413 | |
414 | adr_l \tmp1, __per_cpu_offset | |
415 | ldr w\tmp2, [\tsk, #TSK_CPU] | |
416 | ldr \tmp1, [\tmp1, \tmp2, lsl #3] | |
417 | set_this_cpu_offset \tmp1 | |
7d7b720a MV |
418 | .endm |
419 | ||
034edabe | 420 | /* |
a871d354 | 421 | * The following fragment of code is executed with the MMU enabled. |
b929fe32 AB |
422 | * |
423 | * x0 = __PHYS_OFFSET | |
034edabe | 424 | */ |
c63d9f82 | 425 | SYM_FUNC_START_LOCAL(__primary_switched) |
8e334d72 | 426 | adr_l x4, init_task |
3d8c1a01 | 427 | init_cpu_task x4, x5, x6 |
60699ba1 | 428 | |
2bf31a4a AB |
429 | adr_l x8, vectors // load VBAR_EL1 with virtual |
430 | msr vbar_el1, x8 // vector table address | |
431 | isb | |
432 | ||
8e334d72 | 433 | stp x29, x30, [sp, #-16]! |
60699ba1 AB |
434 | mov x29, sp |
435 | ||
b929fe32 AB |
436 | str_l x21, __fdt_pointer, x5 // Save FDT pointer |
437 | ||
438 | ldr_l x4, kimage_vaddr // Save the offset between | |
439 | sub x4, x4, x0 // the kernel virtual and | |
440 | str_l x4, kimage_voffset, x5 // physical mappings | |
441 | ||
2a803c4d MR |
442 | // Clear BSS |
443 | adr_l x0, __bss_start | |
444 | mov x1, xzr | |
445 | adr_l x2, __bss_stop | |
446 | sub x2, x2, x0 | |
447 | bl __pi_memset | |
5227cfa7 | 448 | dsb ishst // Make zero page visible to PTW |
2a803c4d | 449 | |
0fea6e9a | 450 | #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) |
39d114dd | 451 | bl kasan_early_init |
f80fb3a3 | 452 | #endif |
f6f0c436 MZ |
453 | mov x0, x21 // pass FDT address in x0 |
454 | bl early_fdt_map // Try mapping the FDT early | |
33200303 | 455 | bl init_feature_override // Parse cpu feature overrides |
f80fb3a3 | 456 | #ifdef CONFIG_RANDOMIZE_BASE |
08cdac61 AB |
457 | tst x23, ~(MIN_KIMG_ALIGN - 1) // already running randomized? |
458 | b.ne 0f | |
f80fb3a3 AB |
459 | bl kaslr_early_init // parse FDT for KASLR options |
460 | cbz x0, 0f // KASLR disabled? just proceed | |
08cdac61 | 461 | orr x23, x23, x0 // record KASLR offset |
60699ba1 AB |
462 | ldp x29, x30, [sp], #16 // we must enable KASLR, return |
463 | ret // to __primary_switch() | |
f80fb3a3 | 464 | 0: |
39d114dd | 465 | #endif |
0c93df96 | 466 | bl switch_to_vhe // Prefer VHE if possible |
8e334d72 | 467 | ldp x29, x30, [sp], #16 |
7d7b720a MV |
468 | bl start_kernel |
469 | ASM_BUG() | |
c63d9f82 | 470 | SYM_FUNC_END(__primary_switched) |
034edabe | 471 | |
6cf9a2dc RDC |
472 | .pushsection ".rodata", "a" |
473 | SYM_DATA_START(kimage_vaddr) | |
120dc60d | 474 | .quad _text |
6cf9a2dc RDC |
475 | SYM_DATA_END(kimage_vaddr) |
476 | EXPORT_SYMBOL(kimage_vaddr) | |
477 | .popsection | |
478 | ||
034edabe LA |
479 | /* |
480 | * end early head section, begin head code that is also used for | |
481 | * hotplug and needs to have the same protections as the text region | |
482 | */ | |
439e70e2 | 483 | .section ".idmap.text","awx" |
f80fb3a3 | 484 | |
9703d9d7 | 485 | /* |
ecbb11ab MR |
486 | * Starting from EL2 or EL1, configure the CPU to execute at the highest |
487 | * reachable EL supported by the kernel in a chosen default state. If dropping | |
488 | * from EL2 to EL1, configure EL2 before configuring EL1. | |
828e9834 | 489 | * |
d87a8e65 MR |
490 | * Since we cannot always rely on ERET synchronizing writes to sysregs (e.g. if |
491 | * SCTLR_ELx.EOS is clear), we place an ISB prior to ERET. | |
828e9834 | 492 | * |
510224c2 | 493 | * Returns either BOOT_CPU_MODE_EL1 or BOOT_CPU_MODE_EL2 in w0 if |
828e9834 | 494 | * booted in EL1 or EL2 respectively. |
9703d9d7 | 495 | */ |
ecbb11ab | 496 | SYM_FUNC_START(init_kernel_el) |
9703d9d7 | 497 | mrs x0, CurrentEL |
974c8e45 | 498 | cmp x0, #CurrentEL_EL2 |
d87a8e65 MR |
499 | b.eq init_el2 |
500 | ||
501 | SYM_INNER_LABEL(init_el1, SYM_L_LOCAL) | |
31a32b49 MZ |
502 | mov_q x0, INIT_SCTLR_EL1_MMU_OFF |
503 | msr sctlr_el1, x0 | |
9cf71728 | 504 | isb |
d87a8e65 MR |
505 | mov_q x0, INIT_PSTATE_EL1 |
506 | msr spsr_el1, x0 | |
507 | msr elr_el1, lr | |
508 | mov w0, #BOOT_CPU_MODE_EL1 | |
509 | eret | |
9703d9d7 | 510 | |
d87a8e65 | 511 | SYM_INNER_LABEL(init_el2, SYM_L_LOCAL) |
78869f0f DB |
512 | mov_q x0, HCR_HOST_NVHE_FLAGS |
513 | msr hcr_el2, x0 | |
22043a3c | 514 | isb |
78869f0f | 515 | |
e2df4641 | 516 | init_el2_state |
22043a3c | 517 | |
712c6ff4 | 518 | /* Hypervisor stub */ |
78869f0f | 519 | adr_l x0, __hyp_stub_vectors |
712c6ff4 | 520 | msr vbar_el2, x0 |
d87a8e65 | 521 | isb |
78869f0f | 522 | |
31a32b49 MZ |
523 | /* |
524 | * Fruity CPUs seem to have HCR_EL2.E2H set to RES1, | |
525 | * making it impossible to start in nVHE mode. Is that | |
526 | * compliant with the architecture? Absolutely not! | |
527 | */ | |
528 | mrs x0, hcr_el2 | |
529 | and x0, x0, #HCR_E2H | |
530 | cbz x0, 1f | |
531 | ||
532 | /* Switching to VHE requires a sane SCTLR_EL1 as a start */ | |
533 | mov_q x0, INIT_SCTLR_EL1_MMU_OFF | |
534 | msr_s SYS_SCTLR_EL12, x0 | |
535 | ||
536 | /* | |
537 | * Force an eret into a helper "function", and let it return | |
538 | * to our original caller... This makes sure that we have | |
539 | * initialised the basic PSTATE state. | |
540 | */ | |
541 | mov x0, #INIT_PSTATE_EL2 | |
542 | msr spsr_el1, x0 | |
543 | adr x0, __cpu_stick_to_vhe | |
544 | msr elr_el1, x0 | |
545 | eret | |
546 | ||
547 | 1: | |
548 | mov_q x0, INIT_SCTLR_EL1_MMU_OFF | |
549 | msr sctlr_el1, x0 | |
550 | ||
9703d9d7 | 551 | msr elr_el2, lr |
d87a8e65 | 552 | mov w0, #BOOT_CPU_MODE_EL2 |
9703d9d7 | 553 | eret |
31a32b49 MZ |
554 | |
555 | __cpu_stick_to_vhe: | |
556 | mov x0, #HVC_VHE_RESTART | |
557 | hvc #0 | |
558 | mov x0, #BOOT_CPU_MODE_EL2 | |
559 | ret | |
ecbb11ab | 560 | SYM_FUNC_END(init_kernel_el) |
9703d9d7 | 561 | |
828e9834 ML |
562 | /* |
563 | * Sets the __boot_cpu_mode flag depending on the CPU boot mode passed | |
510224c2 | 564 | * in w0. See arch/arm64/include/asm/virt.h for more info. |
828e9834 | 565 | */ |
c63d9f82 | 566 | SYM_FUNC_START_LOCAL(set_cpu_boot_mode_flag) |
6f4d57fa | 567 | adr_l x1, __boot_cpu_mode |
23c8a500 | 568 | cmp w0, #BOOT_CPU_MODE_EL2 |
828e9834 ML |
569 | b.ne 1f |
570 | add x1, x1, #4 | |
7957a3db | 571 | 1: str w0, [x1] // Save CPU boot mode |
d0488597 WD |
572 | dmb sy |
573 | dc ivac, x1 // Invalidate potentially stale cache line | |
828e9834 | 574 | ret |
c63d9f82 | 575 | SYM_FUNC_END(set_cpu_boot_mode_flag) |
828e9834 | 576 | |
b6113038 JM |
577 | /* |
578 | * These values are written with the MMU off, but read with the MMU on. | |
579 | * Writers will invalidate the corresponding address, discarding up to a | |
580 | * 'Cache Writeback Granule' (CWG) worth of data. The linker script ensures | |
581 | * sufficient alignment that the CWG doesn't overlap another section. | |
582 | */ | |
583 | .pushsection ".mmuoff.data.write", "aw" | |
f35a9205 MZ |
584 | /* |
585 | * We need to find out the CPU boot mode long after boot, so we need to | |
586 | * store it in a writable variable. | |
587 | * | |
588 | * This is not in .bss, because we set it sufficiently early that the boot-time | |
589 | * zeroing of .bss would clobber it. | |
590 | */ | |
a5d4420b | 591 | SYM_DATA_START(__boot_cpu_mode) |
f35a9205 | 592 | .long BOOT_CPU_MODE_EL2 |
424a3838 | 593 | .long BOOT_CPU_MODE_EL1 |
a5d4420b | 594 | SYM_DATA_END(__boot_cpu_mode) |
b6113038 JM |
595 | /* |
596 | * The booting CPU updates the failed status @__early_cpu_boot_status, | |
597 | * with MMU turned off. | |
598 | */ | |
a5d4420b | 599 | SYM_DATA_START(__early_cpu_boot_status) |
61cf61d8 | 600 | .quad 0 |
a5d4420b | 601 | SYM_DATA_END(__early_cpu_boot_status) |
b6113038 | 602 | |
f35a9205 MZ |
603 | .popsection |
604 | ||
9703d9d7 CM |
605 | /* |
606 | * This provides a "holding pen" for platforms to hold all secondary | |
607 | * cores are held until we're ready for them to initialise. | |
608 | */ | |
c63d9f82 | 609 | SYM_FUNC_START(secondary_holding_pen) |
ecbb11ab | 610 | bl init_kernel_el // w0=cpu_boot_mode |
828e9834 | 611 | bl set_cpu_boot_mode_flag |
9703d9d7 | 612 | mrs x0, mpidr_el1 |
b03cc885 | 613 | mov_q x1, MPIDR_HWID_BITMASK |
0359b0e2 | 614 | and x0, x0, x1 |
b1c98297 | 615 | adr_l x3, secondary_holding_pen_release |
9703d9d7 CM |
616 | pen: ldr x4, [x3] |
617 | cmp x4, x0 | |
618 | b.eq secondary_startup | |
619 | wfe | |
620 | b pen | |
c63d9f82 | 621 | SYM_FUNC_END(secondary_holding_pen) |
652af899 MR |
622 | |
623 | /* | |
624 | * Secondary entry point that jumps straight into the kernel. Only to | |
625 | * be used where CPUs are brought online dynamically by the kernel. | |
626 | */ | |
c63d9f82 | 627 | SYM_FUNC_START(secondary_entry) |
ecbb11ab | 628 | bl init_kernel_el // w0=cpu_boot_mode |
85cc00ea | 629 | bl set_cpu_boot_mode_flag |
652af899 | 630 | b secondary_startup |
c63d9f82 | 631 | SYM_FUNC_END(secondary_entry) |
9703d9d7 | 632 | |
c63d9f82 | 633 | SYM_FUNC_START_LOCAL(secondary_startup) |
9703d9d7 CM |
634 | /* |
635 | * Common entry point for secondary CPUs. | |
636 | */ | |
0c93df96 | 637 | bl switch_to_vhe |
a96a33b1 | 638 | bl __cpu_secondary_check52bitva |
a591ede4 | 639 | bl __cpu_setup // initialise processor |
693d5639 | 640 | adrp x1, swapper_pg_dir |
9dcf7914 AB |
641 | bl __enable_mmu |
642 | ldr x8, =__secondary_switched | |
643 | br x8 | |
c63d9f82 | 644 | SYM_FUNC_END(secondary_startup) |
9703d9d7 | 645 | |
c63d9f82 | 646 | SYM_FUNC_START_LOCAL(__secondary_switched) |
2bf31a4a AB |
647 | adr_l x5, vectors |
648 | msr vbar_el1, x5 | |
649 | isb | |
650 | ||
bb905274 | 651 | adr_l x0, secondary_data |
c02433dd | 652 | ldr x2, [x0, #CPU_BOOT_TASK] |
5b1cfe3a | 653 | cbz x2, __secondary_too_slow |
3305e7f7 | 654 | |
3d8c1a01 | 655 | init_cpu_task x2, x1, x3 |
62a679cb MR |
656 | |
657 | #ifdef CONFIG_ARM64_PTR_AUTH | |
658 | ptrauth_keys_init_cpu x2, x3, x4, x5 | |
659 | #endif | |
660 | ||
7d7b720a MV |
661 | bl secondary_start_kernel |
662 | ASM_BUG() | |
c63d9f82 | 663 | SYM_FUNC_END(__secondary_switched) |
9703d9d7 | 664 | |
c63d9f82 | 665 | SYM_FUNC_START_LOCAL(__secondary_too_slow) |
5b1cfe3a WD |
666 | wfe |
667 | wfi | |
668 | b __secondary_too_slow | |
c63d9f82 | 669 | SYM_FUNC_END(__secondary_too_slow) |
5b1cfe3a | 670 | |
bb905274 SP |
671 | /* |
672 | * The booting CPU updates the failed status @__early_cpu_boot_status, | |
673 | * with MMU turned off. | |
674 | * | |
675 | * update_early_cpu_boot_status tmp, status | |
676 | * - Corrupts tmp1, tmp2 | |
677 | * - Writes 'status' to __early_cpu_boot_status and makes sure | |
678 | * it is committed to memory. | |
679 | */ | |
680 | ||
681 | .macro update_early_cpu_boot_status status, tmp1, tmp2 | |
682 | mov \tmp2, #\status | |
adb49070 AB |
683 | adr_l \tmp1, __early_cpu_boot_status |
684 | str \tmp2, [\tmp1] | |
bb905274 SP |
685 | dmb sy |
686 | dc ivac, \tmp1 // Invalidate potentially stale cache line | |
687 | .endm | |
688 | ||
9703d9d7 | 689 | /* |
8b0a9575 | 690 | * Enable the MMU. |
9703d9d7 | 691 | * |
8b0a9575 | 692 | * x0 = SCTLR_EL1 value for turning on the MMU. |
693d5639 | 693 | * x1 = TTBR1_EL1 value |
8b0a9575 | 694 | * |
9dcf7914 AB |
695 | * Returns to the caller via x30/lr. This requires the caller to be covered |
696 | * by the .idmap.text section. | |
4bf8b96e SP |
697 | * |
698 | * Checks if the selected granule size is supported by the CPU. | |
699 | * If it isn't, park the CPU | |
9703d9d7 | 700 | */ |
c63d9f82 | 701 | SYM_FUNC_START(__enable_mmu) |
693d5639 JY |
702 | mrs x2, ID_AA64MMFR0_EL1 |
703 | ubfx x2, x2, #ID_AA64MMFR0_TGRAN_SHIFT, 4 | |
26f55386 JM |
704 | cmp x2, #ID_AA64MMFR0_TGRAN_SUPPORTED_MIN |
705 | b.lt __no_granule_support | |
706 | cmp x2, #ID_AA64MMFR0_TGRAN_SUPPORTED_MAX | |
707 | b.gt __no_granule_support | |
693d5639 JY |
708 | update_early_cpu_boot_status 0, x2, x3 |
709 | adrp x2, idmap_pg_dir | |
710 | phys_to_ttbr x1, x1 | |
711 | phys_to_ttbr x2, x2 | |
712 | msr ttbr0_el1, x2 // load TTBR0 | |
c812026c | 713 | offset_ttbr1 x1, x3 |
693d5639 | 714 | msr ttbr1_el1, x1 // load TTBR1 |
9703d9d7 | 715 | isb |
8cc8a324 MZ |
716 | |
717 | set_sctlr_el1 x0 | |
718 | ||
9dcf7914 | 719 | ret |
c63d9f82 | 720 | SYM_FUNC_END(__enable_mmu) |
4bf8b96e | 721 | |
c63d9f82 | 722 | SYM_FUNC_START(__cpu_secondary_check52bitva) |
b6d00d47 | 723 | #ifdef CONFIG_ARM64_VA_BITS_52 |
2c624fe6 | 724 | ldr_l x0, vabits_actual |
a96a33b1 SC |
725 | cmp x0, #52 |
726 | b.ne 2f | |
727 | ||
728 | mrs_s x0, SYS_ID_AA64MMFR2_EL1 | |
729 | and x0, x0, #(0xf << ID_AA64MMFR2_LVA_SHIFT) | |
730 | cbnz x0, 2f | |
731 | ||
66f16a24 WD |
732 | update_early_cpu_boot_status \ |
733 | CPU_STUCK_IN_KERNEL | CPU_STUCK_REASON_52_BIT_VA, x0, x1 | |
a96a33b1 SC |
734 | 1: wfe |
735 | wfi | |
736 | b 1b | |
737 | ||
738 | #endif | |
739 | 2: ret | |
c63d9f82 | 740 | SYM_FUNC_END(__cpu_secondary_check52bitva) |
a96a33b1 | 741 | |
c63d9f82 | 742 | SYM_FUNC_START_LOCAL(__no_granule_support) |
bb905274 | 743 | /* Indicate that this CPU can't boot and is stuck in the kernel */ |
66f16a24 WD |
744 | update_early_cpu_boot_status \ |
745 | CPU_STUCK_IN_KERNEL | CPU_STUCK_REASON_NO_GRAN, x1, x2 | |
bb905274 | 746 | 1: |
4bf8b96e | 747 | wfe |
bb905274 | 748 | wfi |
3c5e9f23 | 749 | b 1b |
c63d9f82 | 750 | SYM_FUNC_END(__no_granule_support) |
e5ebeec8 | 751 | |
0cd3defe | 752 | #ifdef CONFIG_RELOCATABLE |
c63d9f82 | 753 | SYM_FUNC_START_LOCAL(__relocate_kernel) |
0cd3defe AB |
754 | /* |
755 | * Iterate over each entry in the relocation table, and apply the | |
756 | * relocations in place. | |
757 | */ | |
0cd3defe AB |
758 | ldr w9, =__rela_offset // offset to reloc table |
759 | ldr w10, =__rela_size // size of reloc table | |
760 | ||
b03cc885 | 761 | mov_q x11, KIMAGE_VADDR // default virtual offset |
0cd3defe | 762 | add x11, x11, x23 // actual virtual offset |
0cd3defe AB |
763 | add x9, x9, x11 // __va(.rela) |
764 | add x10, x9, x10 // __va(.rela) + sizeof(.rela) | |
765 | ||
766 | 0: cmp x9, x10 | |
08cc55b2 | 767 | b.hs 1f |
5cf896fb PC |
768 | ldp x12, x13, [x9], #24 |
769 | ldr x14, [x9, #-8] | |
770 | cmp w13, #R_AARCH64_RELATIVE | |
08cc55b2 | 771 | b.ne 0b |
5cf896fb PC |
772 | add x14, x14, x23 // relocate |
773 | str x14, [x12, x23] | |
0cd3defe | 774 | b 0b |
5cf896fb PC |
775 | |
776 | 1: | |
777 | #ifdef CONFIG_RELR | |
778 | /* | |
779 | * Apply RELR relocations. | |
780 | * | |
781 | * RELR is a compressed format for storing relative relocations. The | |
782 | * encoded sequence of entries looks like: | |
783 | * [ AAAAAAAA BBBBBBB1 BBBBBBB1 ... AAAAAAAA BBBBBB1 ... ] | |
784 | * | |
785 | * i.e. start with an address, followed by any number of bitmaps. The | |
786 | * address entry encodes 1 relocation. The subsequent bitmap entries | |
787 | * encode up to 63 relocations each, at subsequent offsets following | |
788 | * the last address entry. | |
789 | * | |
790 | * The bitmap entries must have 1 in the least significant bit. The | |
791 | * assumption here is that an address cannot have 1 in lsb. Odd | |
792 | * addresses are not supported. Any odd addresses are stored in the RELA | |
793 | * section, which is handled above. | |
794 | * | |
795 | * Excluding the least significant bit in the bitmap, each non-zero | |
796 | * bit in the bitmap represents a relocation to be applied to | |
797 | * a corresponding machine word that follows the base address | |
798 | * word. The second least significant bit represents the machine | |
799 | * word immediately following the initial address, and each bit | |
800 | * that follows represents the next word, in linear order. As such, | |
801 | * a single bitmap can encode up to 63 relocations in a 64-bit object. | |
802 | * | |
803 | * In this implementation we store the address of the next RELR table | |
804 | * entry in x9, the address being relocated by the current address or | |
805 | * bitmap entry in x13 and the address being relocated by the current | |
806 | * bit in x14. | |
807 | * | |
808 | * Because addends are stored in place in the binary, RELR relocations | |
809 | * cannot be applied idempotently. We use x24 to keep track of the | |
810 | * currently applied displacement so that we can correctly relocate if | |
811 | * __relocate_kernel is called twice with non-zero displacements (i.e. | |
812 | * if there is both a physical misalignment and a KASLR displacement). | |
813 | */ | |
814 | ldr w9, =__relr_offset // offset to reloc table | |
815 | ldr w10, =__relr_size // size of reloc table | |
816 | add x9, x9, x11 // __va(.relr) | |
817 | add x10, x9, x10 // __va(.relr) + sizeof(.relr) | |
818 | ||
819 | sub x15, x23, x24 // delta from previous offset | |
820 | cbz x15, 7f // nothing to do if unchanged | |
821 | mov x24, x23 // save new offset | |
822 | ||
823 | 2: cmp x9, x10 | |
824 | b.hs 7f | |
825 | ldr x11, [x9], #8 | |
826 | tbnz x11, #0, 3f // branch to handle bitmaps | |
827 | add x13, x11, x23 | |
828 | ldr x12, [x13] // relocate address entry | |
829 | add x12, x12, x15 | |
830 | str x12, [x13], #8 // adjust to start of bitmap | |
831 | b 2b | |
832 | ||
833 | 3: mov x14, x13 | |
834 | 4: lsr x11, x11, #1 | |
835 | cbz x11, 6f | |
836 | tbz x11, #0, 5f // skip bit if not set | |
837 | ldr x12, [x14] // relocate bit | |
838 | add x12, x12, x15 | |
839 | str x12, [x14] | |
840 | ||
841 | 5: add x14, x14, #8 // move to next bit's address | |
842 | b 4b | |
843 | ||
844 | 6: /* | |
845 | * Move to the next bitmap's address. 8 is the word size, and 63 is the | |
846 | * number of significant bits in a bitmap entry. | |
847 | */ | |
848 | add x13, x13, #(8 * 63) | |
849 | b 2b | |
850 | ||
851 | 7: | |
852 | #endif | |
853 | ret | |
854 | ||
c63d9f82 | 855 | SYM_FUNC_END(__relocate_kernel) |
3c5e9f23 | 856 | #endif |
0cd3defe | 857 | |
c63d9f82 | 858 | SYM_FUNC_START_LOCAL(__primary_switch) |
3c5e9f23 AB |
859 | #ifdef CONFIG_RANDOMIZE_BASE |
860 | mov x19, x0 // preserve new SCTLR_EL1 value | |
861 | mrs x20, sctlr_el1 // preserve old SCTLR_EL1 value | |
862 | #endif | |
863 | ||
2b5548b6 | 864 | adrp x1, init_pg_dir |
9dcf7914 | 865 | bl __enable_mmu |
3c5e9f23 | 866 | #ifdef CONFIG_RELOCATABLE |
5cf896fb PC |
867 | #ifdef CONFIG_RELR |
868 | mov x24, #0 // no RELR displacement yet | |
869 | #endif | |
3c5e9f23 AB |
870 | bl __relocate_kernel |
871 | #ifdef CONFIG_RANDOMIZE_BASE | |
872 | ldr x8, =__primary_switched | |
b929fe32 | 873 | adrp x0, __PHYS_OFFSET |
3c5e9f23 AB |
874 | blr x8 |
875 | ||
876 | /* | |
877 | * If we return here, we have a KASLR displacement in x23 which we need | |
878 | * to take into account by discarding the current kernel mapping and | |
879 | * creating a new one. | |
880 | */ | |
3060e9f0 | 881 | pre_disable_mmu_workaround |
3c5e9f23 AB |
882 | msr sctlr_el1, x20 // disable the MMU |
883 | isb | |
884 | bl __create_page_tables // recreate kernel mapping | |
885 | ||
886 | tlbi vmalle1 // Remove any stale TLB entries | |
887 | dsb nsh | |
9d41053e | 888 | isb |
3c5e9f23 | 889 | |
8cc8a324 | 890 | set_sctlr_el1 x19 // re-enable the MMU |
3c5e9f23 AB |
891 | |
892 | bl __relocate_kernel | |
893 | #endif | |
0cd3defe AB |
894 | #endif |
895 | ldr x8, =__primary_switched | |
b929fe32 | 896 | adrp x0, __PHYS_OFFSET |
0cd3defe | 897 | br x8 |
c63d9f82 | 898 | SYM_FUNC_END(__primary_switch) |