]>
Commit | Line | Data |
---|---|---|
7744ccdb TL |
1 | /* |
2 | * AMD Memory Encryption Support | |
3 | * | |
4 | * Copyright (C) 2016 Advanced Micro Devices, Inc. | |
5 | * | |
6 | * Author: Tom Lendacky <thomas.lendacky@amd.com> | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or modify | |
9 | * it under the terms of the GNU General Public License version 2 as | |
10 | * published by the Free Software Foundation. | |
11 | */ | |
12 | ||
13 | #include <linux/linkage.h> | |
5868f365 | 14 | #include <linux/init.h> |
21729f81 | 15 | #include <linux/mm.h> |
c7753208 TL |
16 | #include <linux/dma-mapping.h> |
17 | #include <linux/swiotlb.h> | |
7744ccdb | 18 | |
7f8b7e7f TL |
19 | #include <asm/tlbflush.h> |
20 | #include <asm/fixmap.h> | |
b9d05200 TL |
21 | #include <asm/setup.h> |
22 | #include <asm/bootparam.h> | |
c7753208 | 23 | #include <asm/set_memory.h> |
6ebcb060 TL |
24 | #include <asm/cacheflush.h> |
25 | #include <asm/sections.h> | |
7f8b7e7f | 26 | |
7744ccdb TL |
27 | /* |
28 | * Since SME related variables are set early in the boot process they must | |
29 | * reside in the .data section so as not to be zeroed out when the .bss | |
30 | * section is later cleared. | |
31 | */ | |
32 | unsigned long sme_me_mask __section(.data) = 0; | |
33 | EXPORT_SYMBOL_GPL(sme_me_mask); | |
5868f365 | 34 | |
7f8b7e7f TL |
35 | /* Buffer used for early in-place encryption by BSP, no locking needed */ |
36 | static char sme_early_buffer[PAGE_SIZE] __aligned(PAGE_SIZE); | |
37 | ||
38 | /* | |
39 | * This routine does not change the underlying encryption setting of the | |
40 | * page(s) that map this memory. It assumes that eventually the memory is | |
41 | * meant to be accessed as either encrypted or decrypted but the contents | |
42 | * are currently not in the desired state. | |
43 | * | |
44 | * This routine follows the steps outlined in the AMD64 Architecture | |
45 | * Programmer's Manual Volume 2, Section 7.10.8 Encrypt-in-Place. | |
46 | */ | |
47 | static void __init __sme_early_enc_dec(resource_size_t paddr, | |
48 | unsigned long size, bool enc) | |
49 | { | |
50 | void *src, *dst; | |
51 | size_t len; | |
52 | ||
53 | if (!sme_me_mask) | |
54 | return; | |
55 | ||
56 | local_flush_tlb(); | |
57 | wbinvd(); | |
58 | ||
59 | /* | |
60 | * There are limited number of early mapping slots, so map (at most) | |
61 | * one page at time. | |
62 | */ | |
63 | while (size) { | |
64 | len = min_t(size_t, sizeof(sme_early_buffer), size); | |
65 | ||
66 | /* | |
67 | * Create mappings for the current and desired format of | |
68 | * the memory. Use a write-protected mapping for the source. | |
69 | */ | |
70 | src = enc ? early_memremap_decrypted_wp(paddr, len) : | |
71 | early_memremap_encrypted_wp(paddr, len); | |
72 | ||
73 | dst = enc ? early_memremap_encrypted(paddr, len) : | |
74 | early_memremap_decrypted(paddr, len); | |
75 | ||
76 | /* | |
77 | * If a mapping can't be obtained to perform the operation, | |
78 | * then eventual access of that area in the desired mode | |
79 | * will cause a crash. | |
80 | */ | |
81 | BUG_ON(!src || !dst); | |
82 | ||
83 | /* | |
84 | * Use a temporary buffer, of cache-line multiple size, to | |
85 | * avoid data corruption as documented in the APM. | |
86 | */ | |
87 | memcpy(sme_early_buffer, src, len); | |
88 | memcpy(dst, sme_early_buffer, len); | |
89 | ||
90 | early_memunmap(dst, len); | |
91 | early_memunmap(src, len); | |
92 | ||
93 | paddr += len; | |
94 | size -= len; | |
95 | } | |
96 | } | |
97 | ||
98 | void __init sme_early_encrypt(resource_size_t paddr, unsigned long size) | |
99 | { | |
100 | __sme_early_enc_dec(paddr, size, true); | |
101 | } | |
102 | ||
103 | void __init sme_early_decrypt(resource_size_t paddr, unsigned long size) | |
104 | { | |
105 | __sme_early_enc_dec(paddr, size, false); | |
106 | } | |
107 | ||
b9d05200 TL |
108 | static void __init __sme_early_map_unmap_mem(void *vaddr, unsigned long size, |
109 | bool map) | |
110 | { | |
111 | unsigned long paddr = (unsigned long)vaddr - __PAGE_OFFSET; | |
112 | pmdval_t pmd_flags, pmd; | |
113 | ||
114 | /* Use early_pmd_flags but remove the encryption mask */ | |
115 | pmd_flags = __sme_clr(early_pmd_flags); | |
116 | ||
117 | do { | |
118 | pmd = map ? (paddr & PMD_MASK) + pmd_flags : 0; | |
119 | __early_make_pgtable((unsigned long)vaddr, pmd); | |
120 | ||
121 | vaddr += PMD_SIZE; | |
122 | paddr += PMD_SIZE; | |
123 | size = (size <= PMD_SIZE) ? 0 : size - PMD_SIZE; | |
124 | } while (size); | |
125 | ||
126 | __native_flush_tlb(); | |
127 | } | |
128 | ||
129 | void __init sme_unmap_bootdata(char *real_mode_data) | |
130 | { | |
131 | struct boot_params *boot_data; | |
132 | unsigned long cmdline_paddr; | |
133 | ||
134 | if (!sme_active()) | |
135 | return; | |
136 | ||
137 | /* Get the command line address before unmapping the real_mode_data */ | |
138 | boot_data = (struct boot_params *)real_mode_data; | |
139 | cmdline_paddr = boot_data->hdr.cmd_line_ptr | ((u64)boot_data->ext_cmd_line_ptr << 32); | |
140 | ||
141 | __sme_early_map_unmap_mem(real_mode_data, sizeof(boot_params), false); | |
142 | ||
143 | if (!cmdline_paddr) | |
144 | return; | |
145 | ||
146 | __sme_early_map_unmap_mem(__va(cmdline_paddr), COMMAND_LINE_SIZE, false); | |
147 | } | |
148 | ||
149 | void __init sme_map_bootdata(char *real_mode_data) | |
150 | { | |
151 | struct boot_params *boot_data; | |
152 | unsigned long cmdline_paddr; | |
153 | ||
154 | if (!sme_active()) | |
155 | return; | |
156 | ||
157 | __sme_early_map_unmap_mem(real_mode_data, sizeof(boot_params), true); | |
158 | ||
159 | /* Get the command line address after mapping the real_mode_data */ | |
160 | boot_data = (struct boot_params *)real_mode_data; | |
161 | cmdline_paddr = boot_data->hdr.cmd_line_ptr | ((u64)boot_data->ext_cmd_line_ptr << 32); | |
162 | ||
163 | if (!cmdline_paddr) | |
164 | return; | |
165 | ||
166 | __sme_early_map_unmap_mem(__va(cmdline_paddr), COMMAND_LINE_SIZE, true); | |
167 | } | |
168 | ||
21729f81 TL |
169 | void __init sme_early_init(void) |
170 | { | |
171 | unsigned int i; | |
172 | ||
173 | if (!sme_me_mask) | |
174 | return; | |
175 | ||
176 | early_pmd_flags = __sme_set(early_pmd_flags); | |
177 | ||
178 | __supported_pte_mask = __sme_set(__supported_pte_mask); | |
179 | ||
180 | /* Update the protection map with memory encryption mask */ | |
181 | for (i = 0; i < ARRAY_SIZE(protection_map); i++) | |
182 | protection_map[i] = pgprot_encrypted(protection_map[i]); | |
183 | } | |
184 | ||
c7753208 TL |
185 | /* Architecture __weak replacement functions */ |
186 | void __init mem_encrypt_init(void) | |
187 | { | |
188 | if (!sme_me_mask) | |
189 | return; | |
190 | ||
191 | /* Call into SWIOTLB to update the SWIOTLB DMA buffers */ | |
192 | swiotlb_update_mem_attributes(); | |
193 | } | |
194 | ||
195 | void swiotlb_set_mem_attributes(void *vaddr, unsigned long size) | |
196 | { | |
197 | WARN(PAGE_ALIGN(size) != size, | |
198 | "size is not page-aligned (%#lx)\n", size); | |
199 | ||
200 | /* Make the SWIOTLB buffer area decrypted */ | |
201 | set_memory_decrypted((unsigned long)vaddr, size >> PAGE_SHIFT); | |
202 | } | |
203 | ||
6ebcb060 TL |
204 | static void __init sme_clear_pgd(pgd_t *pgd_base, unsigned long start, |
205 | unsigned long end) | |
206 | { | |
207 | unsigned long pgd_start, pgd_end, pgd_size; | |
208 | pgd_t *pgd_p; | |
209 | ||
210 | pgd_start = start & PGDIR_MASK; | |
211 | pgd_end = end & PGDIR_MASK; | |
212 | ||
213 | pgd_size = (((pgd_end - pgd_start) / PGDIR_SIZE) + 1); | |
214 | pgd_size *= sizeof(pgd_t); | |
215 | ||
216 | pgd_p = pgd_base + pgd_index(start); | |
217 | ||
218 | memset(pgd_p, 0, pgd_size); | |
219 | } | |
220 | ||
221 | #define PGD_FLAGS _KERNPG_TABLE_NOENC | |
222 | #define P4D_FLAGS _KERNPG_TABLE_NOENC | |
223 | #define PUD_FLAGS _KERNPG_TABLE_NOENC | |
224 | #define PMD_FLAGS (__PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL) | |
225 | ||
226 | static void __init *sme_populate_pgd(pgd_t *pgd_base, void *pgtable_area, | |
227 | unsigned long vaddr, pmdval_t pmd_val) | |
228 | { | |
229 | pgd_t *pgd_p; | |
230 | p4d_t *p4d_p; | |
231 | pud_t *pud_p; | |
232 | pmd_t *pmd_p; | |
233 | ||
234 | pgd_p = pgd_base + pgd_index(vaddr); | |
235 | if (native_pgd_val(*pgd_p)) { | |
236 | if (IS_ENABLED(CONFIG_X86_5LEVEL)) | |
237 | p4d_p = (p4d_t *)(native_pgd_val(*pgd_p) & ~PTE_FLAGS_MASK); | |
238 | else | |
239 | pud_p = (pud_t *)(native_pgd_val(*pgd_p) & ~PTE_FLAGS_MASK); | |
240 | } else { | |
241 | pgd_t pgd; | |
242 | ||
243 | if (IS_ENABLED(CONFIG_X86_5LEVEL)) { | |
244 | p4d_p = pgtable_area; | |
245 | memset(p4d_p, 0, sizeof(*p4d_p) * PTRS_PER_P4D); | |
246 | pgtable_area += sizeof(*p4d_p) * PTRS_PER_P4D; | |
247 | ||
248 | pgd = native_make_pgd((pgdval_t)p4d_p + PGD_FLAGS); | |
249 | } else { | |
250 | pud_p = pgtable_area; | |
251 | memset(pud_p, 0, sizeof(*pud_p) * PTRS_PER_PUD); | |
252 | pgtable_area += sizeof(*pud_p) * PTRS_PER_PUD; | |
253 | ||
254 | pgd = native_make_pgd((pgdval_t)pud_p + PGD_FLAGS); | |
255 | } | |
256 | native_set_pgd(pgd_p, pgd); | |
257 | } | |
258 | ||
259 | if (IS_ENABLED(CONFIG_X86_5LEVEL)) { | |
260 | p4d_p += p4d_index(vaddr); | |
261 | if (native_p4d_val(*p4d_p)) { | |
262 | pud_p = (pud_t *)(native_p4d_val(*p4d_p) & ~PTE_FLAGS_MASK); | |
263 | } else { | |
264 | p4d_t p4d; | |
265 | ||
266 | pud_p = pgtable_area; | |
267 | memset(pud_p, 0, sizeof(*pud_p) * PTRS_PER_PUD); | |
268 | pgtable_area += sizeof(*pud_p) * PTRS_PER_PUD; | |
269 | ||
270 | p4d = native_make_p4d((pudval_t)pud_p + P4D_FLAGS); | |
271 | native_set_p4d(p4d_p, p4d); | |
272 | } | |
273 | } | |
274 | ||
275 | pud_p += pud_index(vaddr); | |
276 | if (native_pud_val(*pud_p)) { | |
277 | if (native_pud_val(*pud_p) & _PAGE_PSE) | |
278 | goto out; | |
279 | ||
280 | pmd_p = (pmd_t *)(native_pud_val(*pud_p) & ~PTE_FLAGS_MASK); | |
281 | } else { | |
282 | pud_t pud; | |
283 | ||
284 | pmd_p = pgtable_area; | |
285 | memset(pmd_p, 0, sizeof(*pmd_p) * PTRS_PER_PMD); | |
286 | pgtable_area += sizeof(*pmd_p) * PTRS_PER_PMD; | |
287 | ||
288 | pud = native_make_pud((pmdval_t)pmd_p + PUD_FLAGS); | |
289 | native_set_pud(pud_p, pud); | |
290 | } | |
291 | ||
292 | pmd_p += pmd_index(vaddr); | |
293 | if (!native_pmd_val(*pmd_p) || !(native_pmd_val(*pmd_p) & _PAGE_PSE)) | |
294 | native_set_pmd(pmd_p, native_make_pmd(pmd_val)); | |
295 | ||
296 | out: | |
297 | return pgtable_area; | |
298 | } | |
299 | ||
300 | static unsigned long __init sme_pgtable_calc(unsigned long len) | |
301 | { | |
302 | unsigned long p4d_size, pud_size, pmd_size; | |
303 | unsigned long total; | |
304 | ||
305 | /* | |
306 | * Perform a relatively simplistic calculation of the pagetable | |
307 | * entries that are needed. That mappings will be covered by 2MB | |
308 | * PMD entries so we can conservatively calculate the required | |
309 | * number of P4D, PUD and PMD structures needed to perform the | |
310 | * mappings. Incrementing the count for each covers the case where | |
311 | * the addresses cross entries. | |
312 | */ | |
313 | if (IS_ENABLED(CONFIG_X86_5LEVEL)) { | |
314 | p4d_size = (ALIGN(len, PGDIR_SIZE) / PGDIR_SIZE) + 1; | |
315 | p4d_size *= sizeof(p4d_t) * PTRS_PER_P4D; | |
316 | pud_size = (ALIGN(len, P4D_SIZE) / P4D_SIZE) + 1; | |
317 | pud_size *= sizeof(pud_t) * PTRS_PER_PUD; | |
318 | } else { | |
319 | p4d_size = 0; | |
320 | pud_size = (ALIGN(len, PGDIR_SIZE) / PGDIR_SIZE) + 1; | |
321 | pud_size *= sizeof(pud_t) * PTRS_PER_PUD; | |
322 | } | |
323 | pmd_size = (ALIGN(len, PUD_SIZE) / PUD_SIZE) + 1; | |
324 | pmd_size *= sizeof(pmd_t) * PTRS_PER_PMD; | |
325 | ||
326 | total = p4d_size + pud_size + pmd_size; | |
327 | ||
328 | /* | |
329 | * Now calculate the added pagetable structures needed to populate | |
330 | * the new pagetables. | |
331 | */ | |
332 | if (IS_ENABLED(CONFIG_X86_5LEVEL)) { | |
333 | p4d_size = ALIGN(total, PGDIR_SIZE) / PGDIR_SIZE; | |
334 | p4d_size *= sizeof(p4d_t) * PTRS_PER_P4D; | |
335 | pud_size = ALIGN(total, P4D_SIZE) / P4D_SIZE; | |
336 | pud_size *= sizeof(pud_t) * PTRS_PER_PUD; | |
337 | } else { | |
338 | p4d_size = 0; | |
339 | pud_size = ALIGN(total, PGDIR_SIZE) / PGDIR_SIZE; | |
340 | pud_size *= sizeof(pud_t) * PTRS_PER_PUD; | |
341 | } | |
342 | pmd_size = ALIGN(total, PUD_SIZE) / PUD_SIZE; | |
343 | pmd_size *= sizeof(pmd_t) * PTRS_PER_PMD; | |
344 | ||
345 | total += p4d_size + pud_size + pmd_size; | |
346 | ||
347 | return total; | |
348 | } | |
349 | ||
5868f365 TL |
350 | void __init sme_encrypt_kernel(void) |
351 | { | |
6ebcb060 TL |
352 | unsigned long workarea_start, workarea_end, workarea_len; |
353 | unsigned long execute_start, execute_end, execute_len; | |
354 | unsigned long kernel_start, kernel_end, kernel_len; | |
355 | unsigned long pgtable_area_len; | |
356 | unsigned long paddr, pmd_flags; | |
357 | unsigned long decrypted_base; | |
358 | void *pgtable_area; | |
359 | pgd_t *pgd; | |
360 | ||
361 | if (!sme_active()) | |
362 | return; | |
363 | ||
364 | /* | |
365 | * Prepare for encrypting the kernel by building new pagetables with | |
366 | * the necessary attributes needed to encrypt the kernel in place. | |
367 | * | |
368 | * One range of virtual addresses will map the memory occupied | |
369 | * by the kernel as encrypted. | |
370 | * | |
371 | * Another range of virtual addresses will map the memory occupied | |
372 | * by the kernel as decrypted and write-protected. | |
373 | * | |
374 | * The use of write-protect attribute will prevent any of the | |
375 | * memory from being cached. | |
376 | */ | |
377 | ||
378 | /* Physical addresses gives us the identity mapped virtual addresses */ | |
379 | kernel_start = __pa_symbol(_text); | |
380 | kernel_end = ALIGN(__pa_symbol(_end), PMD_PAGE_SIZE); | |
381 | kernel_len = kernel_end - kernel_start; | |
382 | ||
383 | /* Set the encryption workarea to be immediately after the kernel */ | |
384 | workarea_start = kernel_end; | |
385 | ||
386 | /* | |
387 | * Calculate required number of workarea bytes needed: | |
388 | * executable encryption area size: | |
389 | * stack page (PAGE_SIZE) | |
390 | * encryption routine page (PAGE_SIZE) | |
391 | * intermediate copy buffer (PMD_PAGE_SIZE) | |
392 | * pagetable structures for the encryption of the kernel | |
393 | * pagetable structures for workarea (in case not currently mapped) | |
394 | */ | |
395 | execute_start = workarea_start; | |
396 | execute_end = execute_start + (PAGE_SIZE * 2) + PMD_PAGE_SIZE; | |
397 | execute_len = execute_end - execute_start; | |
398 | ||
399 | /* | |
400 | * One PGD for both encrypted and decrypted mappings and a set of | |
401 | * PUDs and PMDs for each of the encrypted and decrypted mappings. | |
402 | */ | |
403 | pgtable_area_len = sizeof(pgd_t) * PTRS_PER_PGD; | |
404 | pgtable_area_len += sme_pgtable_calc(execute_end - kernel_start) * 2; | |
405 | ||
406 | /* PUDs and PMDs needed in the current pagetables for the workarea */ | |
407 | pgtable_area_len += sme_pgtable_calc(execute_len + pgtable_area_len); | |
408 | ||
409 | /* | |
410 | * The total workarea includes the executable encryption area and | |
411 | * the pagetable area. | |
412 | */ | |
413 | workarea_len = execute_len + pgtable_area_len; | |
414 | workarea_end = workarea_start + workarea_len; | |
415 | ||
416 | /* | |
417 | * Set the address to the start of where newly created pagetable | |
418 | * structures (PGDs, PUDs and PMDs) will be allocated. New pagetable | |
419 | * structures are created when the workarea is added to the current | |
420 | * pagetables and when the new encrypted and decrypted kernel | |
421 | * mappings are populated. | |
422 | */ | |
423 | pgtable_area = (void *)execute_end; | |
424 | ||
425 | /* | |
426 | * Make sure the current pagetable structure has entries for | |
427 | * addressing the workarea. | |
428 | */ | |
429 | pgd = (pgd_t *)native_read_cr3_pa(); | |
430 | paddr = workarea_start; | |
431 | while (paddr < workarea_end) { | |
432 | pgtable_area = sme_populate_pgd(pgd, pgtable_area, | |
433 | paddr, | |
434 | paddr + PMD_FLAGS); | |
435 | ||
436 | paddr += PMD_PAGE_SIZE; | |
437 | } | |
438 | ||
439 | /* Flush the TLB - no globals so cr3 is enough */ | |
440 | native_write_cr3(__native_read_cr3()); | |
441 | ||
442 | /* | |
443 | * A new pagetable structure is being built to allow for the kernel | |
444 | * to be encrypted. It starts with an empty PGD that will then be | |
445 | * populated with new PUDs and PMDs as the encrypted and decrypted | |
446 | * kernel mappings are created. | |
447 | */ | |
448 | pgd = pgtable_area; | |
449 | memset(pgd, 0, sizeof(*pgd) * PTRS_PER_PGD); | |
450 | pgtable_area += sizeof(*pgd) * PTRS_PER_PGD; | |
451 | ||
452 | /* Add encrypted kernel (identity) mappings */ | |
453 | pmd_flags = PMD_FLAGS | _PAGE_ENC; | |
454 | paddr = kernel_start; | |
455 | while (paddr < kernel_end) { | |
456 | pgtable_area = sme_populate_pgd(pgd, pgtable_area, | |
457 | paddr, | |
458 | paddr + pmd_flags); | |
459 | ||
460 | paddr += PMD_PAGE_SIZE; | |
461 | } | |
462 | ||
463 | /* | |
464 | * A different PGD index/entry must be used to get different | |
465 | * pagetable entries for the decrypted mapping. Choose the next | |
466 | * PGD index and convert it to a virtual address to be used as | |
467 | * the base of the mapping. | |
468 | */ | |
469 | decrypted_base = (pgd_index(workarea_end) + 1) & (PTRS_PER_PGD - 1); | |
470 | decrypted_base <<= PGDIR_SHIFT; | |
471 | ||
472 | /* Add decrypted, write-protected kernel (non-identity) mappings */ | |
473 | pmd_flags = (PMD_FLAGS & ~_PAGE_CACHE_MASK) | (_PAGE_PAT | _PAGE_PWT); | |
474 | paddr = kernel_start; | |
475 | while (paddr < kernel_end) { | |
476 | pgtable_area = sme_populate_pgd(pgd, pgtable_area, | |
477 | paddr + decrypted_base, | |
478 | paddr + pmd_flags); | |
479 | ||
480 | paddr += PMD_PAGE_SIZE; | |
481 | } | |
482 | ||
483 | /* Add decrypted workarea mappings to both kernel mappings */ | |
484 | paddr = workarea_start; | |
485 | while (paddr < workarea_end) { | |
486 | pgtable_area = sme_populate_pgd(pgd, pgtable_area, | |
487 | paddr, | |
488 | paddr + PMD_FLAGS); | |
489 | ||
490 | pgtable_area = sme_populate_pgd(pgd, pgtable_area, | |
491 | paddr + decrypted_base, | |
492 | paddr + PMD_FLAGS); | |
493 | ||
494 | paddr += PMD_PAGE_SIZE; | |
495 | } | |
496 | ||
497 | /* Perform the encryption */ | |
498 | sme_encrypt_execute(kernel_start, kernel_start + decrypted_base, | |
499 | kernel_len, workarea_start, (unsigned long)pgd); | |
500 | ||
501 | /* | |
502 | * At this point we are running encrypted. Remove the mappings for | |
503 | * the decrypted areas - all that is needed for this is to remove | |
504 | * the PGD entry/entries. | |
505 | */ | |
506 | sme_clear_pgd(pgd, kernel_start + decrypted_base, | |
507 | kernel_end + decrypted_base); | |
508 | ||
509 | sme_clear_pgd(pgd, workarea_start + decrypted_base, | |
510 | workarea_end + decrypted_base); | |
511 | ||
512 | /* Flush the TLB - no globals so cr3 is enough */ | |
513 | native_write_cr3(__native_read_cr3()); | |
5868f365 TL |
514 | } |
515 | ||
516 | void __init sme_enable(void) | |
517 | { | |
518 | } |