]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
084ee1c6 | 2 | #include <linux/io.h> |
5ff3e2c3 | 3 | #include <linux/slab.h> |
084ee1c6 | 4 | #include <linux/memblock.h> |
163ea3c8 | 5 | #include <linux/mem_encrypt.h> |
084ee1c6 | 6 | |
d1163651 | 7 | #include <asm/set_memory.h> |
084ee1c6 JS |
8 | #include <asm/pgtable.h> |
9 | #include <asm/realmode.h> | |
18bc7bd5 | 10 | #include <asm/tlbflush.h> |
7f6c95e8 | 11 | #include <asm/crash.h> |
084ee1c6 | 12 | |
b429dbf6 | 13 | struct real_mode_header *real_mode_header; |
cda846f1 | 14 | u32 *trampoline_cr4_features; |
084ee1c6 | 15 | |
b234e8a0 TG |
16 | /* Hold the pgd entry used on booting additional CPUs */ |
17 | pgd_t trampoline_pgd_entry; | |
18 | ||
be1c8a44 JR |
19 | void load_trampoline_pgtable(void) |
20 | { | |
21 | #ifdef CONFIG_X86_32 | |
22 | load_cr3(initial_page_table); | |
23 | #else | |
24 | /* | |
25 | * This function is called before exiting to real-mode and that will | |
26 | * fail with CR4.PCIDE still set. | |
27 | */ | |
28 | if (boot_cpu_has(X86_FEATURE_PCID)) | |
29 | cr4_clear_bits(X86_CR4_PCIDE); | |
30 | ||
31 | write_cr3(real_mode_header->trampoline_pgd); | |
32 | #endif | |
33 | ||
34 | /* | |
35 | * The CR3 write above will not flush global TLB entries. | |
36 | * Stale, global entries from previous page tables may still be | |
37 | * present. Flush those stale entries. | |
38 | * | |
39 | * This ensures that memory accessed while running with | |
40 | * trampoline_pgd is *actually* mapped into trampoline_pgd. | |
41 | */ | |
42 | __flush_tlb_all(); | |
43 | } | |
44 | ||
4f7b9226 | 45 | void __init reserve_real_mode(void) |
084ee1c6 JS |
46 | { |
47 | phys_addr_t mem; | |
5ff3e2c3 AL |
48 | size_t size = real_mode_size_needed(); |
49 | ||
50 | if (!size) | |
51 | return; | |
52 | ||
53 | WARN_ON(slab_is_available()); | |
4f7b9226 YL |
54 | |
55 | /* Has to be under 1M so we can execute real-mode AP code. */ | |
56 | mem = memblock_find_in_range(0, 1<<20, size, PAGE_SIZE); | |
5ff3e2c3 AL |
57 | if (!mem) { |
58 | pr_info("No sub-1M memory is available for the trampoline\n"); | |
59 | return; | |
60 | } | |
4f7b9226 | 61 | |
4f7b9226 | 62 | memblock_reserve(mem, size); |
f560bd19 | 63 | set_real_mode_mem(mem); |
7f6c95e8 | 64 | crash_reserve_low_1M(); |
4f7b9226 YL |
65 | } |
66 | ||
d0de0f68 | 67 | static void __init setup_real_mode(void) |
4f7b9226 | 68 | { |
084ee1c6 | 69 | u16 real_mode_seg; |
7306006f | 70 | const u32 *rel; |
084ee1c6 | 71 | u32 count; |
b429dbf6 | 72 | unsigned char *base; |
7306006f | 73 | unsigned long phys_base; |
f37240f1 | 74 | struct trampoline_header *trampoline_header; |
b429dbf6 | 75 | size_t size = PAGE_ALIGN(real_mode_blob_end - real_mode_blob); |
f37240f1 JS |
76 | #ifdef CONFIG_X86_64 |
77 | u64 *trampoline_pgd; | |
638d957b | 78 | u64 efer; |
4914ecc2 | 79 | int i; |
f37240f1 | 80 | #endif |
084ee1c6 | 81 | |
4f7b9226 | 82 | base = (unsigned char *)real_mode_header; |
084ee1c6 | 83 | |
163ea3c8 TL |
84 | /* |
85 | * If SME is active, the trampoline area will need to be in | |
86 | * decrypted memory in order to bring up other processors | |
fcdcd6cd | 87 | * successfully. This is not needed for SEV. |
163ea3c8 | 88 | */ |
fcdcd6cd TL |
89 | if (sme_active()) |
90 | set_memory_decrypted((unsigned long)base, size >> PAGE_SHIFT); | |
163ea3c8 | 91 | |
b429dbf6 | 92 | memcpy(base, real_mode_blob, size); |
084ee1c6 | 93 | |
7306006f PA |
94 | phys_base = __pa(base); |
95 | real_mode_seg = phys_base >> 4; | |
96 | ||
084ee1c6 JS |
97 | rel = (u32 *) real_mode_relocs; |
98 | ||
99 | /* 16-bit segment relocations. */ | |
7306006f PA |
100 | count = *rel++; |
101 | while (count--) { | |
102 | u16 *seg = (u16 *) (base + *rel++); | |
084ee1c6 JS |
103 | *seg = real_mode_seg; |
104 | } | |
105 | ||
106 | /* 32-bit linear relocations. */ | |
7306006f PA |
107 | count = *rel++; |
108 | while (count--) { | |
109 | u32 *ptr = (u32 *) (base + *rel++); | |
110 | *ptr += phys_base; | |
084ee1c6 JS |
111 | } |
112 | ||
f37240f1 JS |
113 | /* Must be perfomed *after* relocation. */ |
114 | trampoline_header = (struct trampoline_header *) | |
115 | __va(real_mode_header->trampoline_header); | |
116 | ||
48927bbb | 117 | #ifdef CONFIG_X86_32 |
fc8d7826 | 118 | trampoline_header->start = __pa_symbol(startup_32_smp); |
f37240f1 | 119 | trampoline_header->gdt_limit = __BOOT_DS + 7; |
fc8d7826 | 120 | trampoline_header->gdt_base = __pa_symbol(boot_gdt); |
48927bbb | 121 | #else |
79603879 PA |
122 | /* |
123 | * Some AMD processors will #GP(0) if EFER.LMA is set in WRMSR | |
124 | * so we need to mask it out. | |
125 | */ | |
638d957b PA |
126 | rdmsrl(MSR_EFER, efer); |
127 | trampoline_header->efer = efer & ~EFER_LMA; | |
cda846f1 | 128 | |
f37240f1 | 129 | trampoline_header->start = (u64) secondary_startup_64; |
cda846f1 | 130 | trampoline_cr4_features = &trampoline_header->cr4; |
18bc7bd5 | 131 | *trampoline_cr4_features = mmu_cr4_features; |
cda846f1 | 132 | |
46d010e0 TL |
133 | trampoline_header->flags = 0; |
134 | if (sme_active()) | |
135 | trampoline_header->flags |= TH_FLAGS_SME_ACTIVE; | |
136 | ||
f37240f1 | 137 | trampoline_pgd = (u64 *) __va(real_mode_header->trampoline_pgd); |
4914ecc2 JR |
138 | |
139 | /* Map the real mode stub as virtual == physical */ | |
b234e8a0 | 140 | trampoline_pgd[0] = trampoline_pgd_entry.pgd; |
4914ecc2 JR |
141 | |
142 | /* | |
143 | * Include the entirety of the kernel mapping into the trampoline | |
144 | * PGD. This way, all mappings present in the normal kernel page | |
145 | * tables are usable while running on trampoline_pgd. | |
146 | */ | |
147 | for (i = pgd_index(__PAGE_OFFSET); i < PTRS_PER_PGD; i++) | |
148 | trampoline_pgd[i] = init_top_pgt[i].pgd; | |
48927bbb | 149 | #endif |
084ee1c6 JS |
150 | } |
151 | ||
152 | /* | |
4f7b9226 | 153 | * reserve_real_mode() gets called very early, to guarantee the |
231b3642 | 154 | * availability of low memory. This is before the proper kernel page |
084ee1c6 | 155 | * tables are set up, so we cannot set page permissions in that |
231b3642 YL |
156 | * function. Also trampoline code will be executed by APs so we |
157 | * need to mark it executable at do_pre_smp_initcalls() at least, | |
158 | * thus run it as a early_initcall(). | |
084ee1c6 | 159 | */ |
d0de0f68 | 160 | static void __init set_real_mode_permissions(void) |
084ee1c6 | 161 | { |
b429dbf6 JS |
162 | unsigned char *base = (unsigned char *) real_mode_header; |
163 | size_t size = PAGE_ALIGN(real_mode_blob_end - real_mode_blob); | |
084ee1c6 | 164 | |
f156ffc4 | 165 | size_t ro_size = |
b429dbf6 JS |
166 | PAGE_ALIGN(real_mode_header->ro_end) - |
167 | __pa(base); | |
f156ffc4 JS |
168 | |
169 | size_t text_size = | |
b429dbf6 JS |
170 | PAGE_ALIGN(real_mode_header->ro_end) - |
171 | real_mode_header->text_start; | |
f156ffc4 JS |
172 | |
173 | unsigned long text_start = | |
b429dbf6 | 174 | (unsigned long) __va(real_mode_header->text_start); |
f156ffc4 | 175 | |
b429dbf6 JS |
176 | set_memory_nx((unsigned long) base, size >> PAGE_SHIFT); |
177 | set_memory_ro((unsigned long) base, ro_size >> PAGE_SHIFT); | |
f156ffc4 | 178 | set_memory_x((unsigned long) text_start, text_size >> PAGE_SHIFT); |
d0de0f68 AL |
179 | } |
180 | ||
181 | static int __init init_real_mode(void) | |
182 | { | |
183 | if (!real_mode_header) | |
184 | panic("Real mode trampoline was not allocated"); | |
185 | ||
186 | setup_real_mode(); | |
187 | set_real_mode_permissions(); | |
f156ffc4 | 188 | |
084ee1c6 JS |
189 | return 0; |
190 | } | |
d0de0f68 | 191 | early_initcall(init_real_mode); |