]>
Commit | Line | Data |
---|---|---|
1 | // SPDX-License-Identifier: GPL-2.0 | |
2 | /* | |
3 | * Copyright (C) 1992 Krishna Balasubramanian and Linus Torvalds | |
4 | * Copyright (C) 1999 Ingo Molnar <mingo@redhat.com> | |
5 | * Copyright (C) 2002 Andi Kleen | |
6 | * | |
7 | * This handles calls from both 32bit and 64bit mode. | |
8 | * | |
9 | * Lock order: | |
10 | * contex.ldt_usr_sem | |
11 | * mmap_lock | |
12 | * context.lock | |
13 | */ | |
14 | ||
15 | #include <linux/errno.h> | |
16 | #include <linux/gfp.h> | |
17 | #include <linux/sched.h> | |
18 | #include <linux/string.h> | |
19 | #include <linux/mm.h> | |
20 | #include <linux/smp.h> | |
21 | #include <linux/syscalls.h> | |
22 | #include <linux/slab.h> | |
23 | #include <linux/vmalloc.h> | |
24 | #include <linux/uaccess.h> | |
25 | ||
26 | #include <asm/ldt.h> | |
27 | #include <asm/tlb.h> | |
28 | #include <asm/desc.h> | |
29 | #include <asm/mmu_context.h> | |
30 | #include <asm/pgtable_areas.h> | |
31 | ||
32 | /* This is a multiple of PAGE_SIZE. */ | |
33 | #define LDT_SLOT_STRIDE (LDT_ENTRIES * LDT_ENTRY_SIZE) | |
34 | ||
35 | static inline void *ldt_slot_va(int slot) | |
36 | { | |
37 | return (void *)(LDT_BASE_ADDR + LDT_SLOT_STRIDE * slot); | |
38 | } | |
39 | ||
40 | void load_mm_ldt(struct mm_struct *mm) | |
41 | { | |
42 | struct ldt_struct *ldt; | |
43 | ||
44 | /* READ_ONCE synchronizes with smp_store_release */ | |
45 | ldt = READ_ONCE(mm->context.ldt); | |
46 | ||
47 | /* | |
48 | * Any change to mm->context.ldt is followed by an IPI to all | |
49 | * CPUs with the mm active. The LDT will not be freed until | |
50 | * after the IPI is handled by all such CPUs. This means that, | |
51 | * if the ldt_struct changes before we return, the values we see | |
52 | * will be safe, and the new values will be loaded before we run | |
53 | * any user code. | |
54 | * | |
55 | * NB: don't try to convert this to use RCU without extreme care. | |
56 | * We would still need IRQs off, because we don't want to change | |
57 | * the local LDT after an IPI loaded a newer value than the one | |
58 | * that we can see. | |
59 | */ | |
60 | ||
61 | if (unlikely(ldt)) { | |
62 | if (static_cpu_has(X86_FEATURE_PTI)) { | |
63 | if (WARN_ON_ONCE((unsigned long)ldt->slot > 1)) { | |
64 | /* | |
65 | * Whoops -- either the new LDT isn't mapped | |
66 | * (if slot == -1) or is mapped into a bogus | |
67 | * slot (if slot > 1). | |
68 | */ | |
69 | clear_LDT(); | |
70 | return; | |
71 | } | |
72 | ||
73 | /* | |
74 | * If page table isolation is enabled, ldt->entries | |
75 | * will not be mapped in the userspace pagetables. | |
76 | * Tell the CPU to access the LDT through the alias | |
77 | * at ldt_slot_va(ldt->slot). | |
78 | */ | |
79 | set_ldt(ldt_slot_va(ldt->slot), ldt->nr_entries); | |
80 | } else { | |
81 | set_ldt(ldt->entries, ldt->nr_entries); | |
82 | } | |
83 | } else { | |
84 | clear_LDT(); | |
85 | } | |
86 | } | |
87 | ||
88 | void switch_ldt(struct mm_struct *prev, struct mm_struct *next) | |
89 | { | |
90 | /* | |
91 | * Load the LDT if either the old or new mm had an LDT. | |
92 | * | |
93 | * An mm will never go from having an LDT to not having an LDT. Two | |
94 | * mms never share an LDT, so we don't gain anything by checking to | |
95 | * see whether the LDT changed. There's also no guarantee that | |
96 | * prev->context.ldt actually matches LDTR, but, if LDTR is non-NULL, | |
97 | * then prev->context.ldt will also be non-NULL. | |
98 | * | |
99 | * If we really cared, we could optimize the case where prev == next | |
100 | * and we're exiting lazy mode. Most of the time, if this happens, | |
101 | * we don't actually need to reload LDTR, but modify_ldt() is mostly | |
102 | * used by legacy code and emulators where we don't need this level of | |
103 | * performance. | |
104 | * | |
105 | * This uses | instead of || because it generates better code. | |
106 | */ | |
107 | if (unlikely((unsigned long)prev->context.ldt | | |
108 | (unsigned long)next->context.ldt)) | |
109 | load_mm_ldt(next); | |
110 | ||
111 | DEBUG_LOCKS_WARN_ON(preemptible()); | |
112 | } | |
113 | ||
114 | static void refresh_ldt_segments(void) | |
115 | { | |
116 | #ifdef CONFIG_X86_64 | |
117 | unsigned short sel; | |
118 | ||
119 | /* | |
120 | * Make sure that the cached DS and ES descriptors match the updated | |
121 | * LDT. | |
122 | */ | |
123 | savesegment(ds, sel); | |
124 | if ((sel & SEGMENT_TI_MASK) == SEGMENT_LDT) | |
125 | loadsegment(ds, sel); | |
126 | ||
127 | savesegment(es, sel); | |
128 | if ((sel & SEGMENT_TI_MASK) == SEGMENT_LDT) | |
129 | loadsegment(es, sel); | |
130 | #endif | |
131 | } | |
132 | ||
133 | /* context.lock is held by the task which issued the smp function call */ | |
134 | static void flush_ldt(void *__mm) | |
135 | { | |
136 | struct mm_struct *mm = __mm; | |
137 | ||
138 | if (this_cpu_read(cpu_tlbstate.loaded_mm) != mm) | |
139 | return; | |
140 | ||
141 | load_mm_ldt(mm); | |
142 | ||
143 | refresh_ldt_segments(); | |
144 | } | |
145 | ||
146 | /* The caller must call finalize_ldt_struct on the result. LDT starts zeroed. */ | |
147 | static struct ldt_struct *alloc_ldt_struct(unsigned int num_entries) | |
148 | { | |
149 | struct ldt_struct *new_ldt; | |
150 | unsigned int alloc_size; | |
151 | ||
152 | if (num_entries > LDT_ENTRIES) | |
153 | return NULL; | |
154 | ||
155 | new_ldt = kmalloc(sizeof(struct ldt_struct), GFP_KERNEL); | |
156 | if (!new_ldt) | |
157 | return NULL; | |
158 | ||
159 | BUILD_BUG_ON(LDT_ENTRY_SIZE != sizeof(struct desc_struct)); | |
160 | alloc_size = num_entries * LDT_ENTRY_SIZE; | |
161 | ||
162 | /* | |
163 | * Xen is very picky: it requires a page-aligned LDT that has no | |
164 | * trailing nonzero bytes in any page that contains LDT descriptors. | |
165 | * Keep it simple: zero the whole allocation and never allocate less | |
166 | * than PAGE_SIZE. | |
167 | */ | |
168 | if (alloc_size > PAGE_SIZE) | |
169 | new_ldt->entries = vzalloc(alloc_size); | |
170 | else | |
171 | new_ldt->entries = (void *)get_zeroed_page(GFP_KERNEL); | |
172 | ||
173 | if (!new_ldt->entries) { | |
174 | kfree(new_ldt); | |
175 | return NULL; | |
176 | } | |
177 | ||
178 | /* The new LDT isn't aliased for PTI yet. */ | |
179 | new_ldt->slot = -1; | |
180 | ||
181 | new_ldt->nr_entries = num_entries; | |
182 | return new_ldt; | |
183 | } | |
184 | ||
185 | #ifdef CONFIG_PAGE_TABLE_ISOLATION | |
186 | ||
187 | static void do_sanity_check(struct mm_struct *mm, | |
188 | bool had_kernel_mapping, | |
189 | bool had_user_mapping) | |
190 | { | |
191 | if (mm->context.ldt) { | |
192 | /* | |
193 | * We already had an LDT. The top-level entry should already | |
194 | * have been allocated and synchronized with the usermode | |
195 | * tables. | |
196 | */ | |
197 | WARN_ON(!had_kernel_mapping); | |
198 | if (boot_cpu_has(X86_FEATURE_PTI)) | |
199 | WARN_ON(!had_user_mapping); | |
200 | } else { | |
201 | /* | |
202 | * This is the first time we're mapping an LDT for this process. | |
203 | * Sync the pgd to the usermode tables. | |
204 | */ | |
205 | WARN_ON(had_kernel_mapping); | |
206 | if (boot_cpu_has(X86_FEATURE_PTI)) | |
207 | WARN_ON(had_user_mapping); | |
208 | } | |
209 | } | |
210 | ||
211 | #ifdef CONFIG_X86_PAE | |
212 | ||
213 | static pmd_t *pgd_to_pmd_walk(pgd_t *pgd, unsigned long va) | |
214 | { | |
215 | p4d_t *p4d; | |
216 | pud_t *pud; | |
217 | ||
218 | if (pgd->pgd == 0) | |
219 | return NULL; | |
220 | ||
221 | p4d = p4d_offset(pgd, va); | |
222 | if (p4d_none(*p4d)) | |
223 | return NULL; | |
224 | ||
225 | pud = pud_offset(p4d, va); | |
226 | if (pud_none(*pud)) | |
227 | return NULL; | |
228 | ||
229 | return pmd_offset(pud, va); | |
230 | } | |
231 | ||
232 | static void map_ldt_struct_to_user(struct mm_struct *mm) | |
233 | { | |
234 | pgd_t *k_pgd = pgd_offset(mm, LDT_BASE_ADDR); | |
235 | pgd_t *u_pgd = kernel_to_user_pgdp(k_pgd); | |
236 | pmd_t *k_pmd, *u_pmd; | |
237 | ||
238 | k_pmd = pgd_to_pmd_walk(k_pgd, LDT_BASE_ADDR); | |
239 | u_pmd = pgd_to_pmd_walk(u_pgd, LDT_BASE_ADDR); | |
240 | ||
241 | if (boot_cpu_has(X86_FEATURE_PTI) && !mm->context.ldt) | |
242 | set_pmd(u_pmd, *k_pmd); | |
243 | } | |
244 | ||
245 | static void sanity_check_ldt_mapping(struct mm_struct *mm) | |
246 | { | |
247 | pgd_t *k_pgd = pgd_offset(mm, LDT_BASE_ADDR); | |
248 | pgd_t *u_pgd = kernel_to_user_pgdp(k_pgd); | |
249 | bool had_kernel, had_user; | |
250 | pmd_t *k_pmd, *u_pmd; | |
251 | ||
252 | k_pmd = pgd_to_pmd_walk(k_pgd, LDT_BASE_ADDR); | |
253 | u_pmd = pgd_to_pmd_walk(u_pgd, LDT_BASE_ADDR); | |
254 | had_kernel = (k_pmd->pmd != 0); | |
255 | had_user = (u_pmd->pmd != 0); | |
256 | ||
257 | do_sanity_check(mm, had_kernel, had_user); | |
258 | } | |
259 | ||
260 | #else /* !CONFIG_X86_PAE */ | |
261 | ||
262 | static void map_ldt_struct_to_user(struct mm_struct *mm) | |
263 | { | |
264 | pgd_t *pgd = pgd_offset(mm, LDT_BASE_ADDR); | |
265 | ||
266 | if (boot_cpu_has(X86_FEATURE_PTI) && !mm->context.ldt) | |
267 | set_pgd(kernel_to_user_pgdp(pgd), *pgd); | |
268 | } | |
269 | ||
270 | static void sanity_check_ldt_mapping(struct mm_struct *mm) | |
271 | { | |
272 | pgd_t *pgd = pgd_offset(mm, LDT_BASE_ADDR); | |
273 | bool had_kernel = (pgd->pgd != 0); | |
274 | bool had_user = (kernel_to_user_pgdp(pgd)->pgd != 0); | |
275 | ||
276 | do_sanity_check(mm, had_kernel, had_user); | |
277 | } | |
278 | ||
279 | #endif /* CONFIG_X86_PAE */ | |
280 | ||
281 | /* | |
282 | * If PTI is enabled, this maps the LDT into the kernelmode and | |
283 | * usermode tables for the given mm. | |
284 | */ | |
285 | static int | |
286 | map_ldt_struct(struct mm_struct *mm, struct ldt_struct *ldt, int slot) | |
287 | { | |
288 | unsigned long va; | |
289 | bool is_vmalloc; | |
290 | spinlock_t *ptl; | |
291 | int i, nr_pages; | |
292 | ||
293 | if (!boot_cpu_has(X86_FEATURE_PTI)) | |
294 | return 0; | |
295 | ||
296 | /* | |
297 | * Any given ldt_struct should have map_ldt_struct() called at most | |
298 | * once. | |
299 | */ | |
300 | WARN_ON(ldt->slot != -1); | |
301 | ||
302 | /* Check if the current mappings are sane */ | |
303 | sanity_check_ldt_mapping(mm); | |
304 | ||
305 | is_vmalloc = is_vmalloc_addr(ldt->entries); | |
306 | ||
307 | nr_pages = DIV_ROUND_UP(ldt->nr_entries * LDT_ENTRY_SIZE, PAGE_SIZE); | |
308 | ||
309 | for (i = 0; i < nr_pages; i++) { | |
310 | unsigned long offset = i << PAGE_SHIFT; | |
311 | const void *src = (char *)ldt->entries + offset; | |
312 | unsigned long pfn; | |
313 | pgprot_t pte_prot; | |
314 | pte_t pte, *ptep; | |
315 | ||
316 | va = (unsigned long)ldt_slot_va(slot) + offset; | |
317 | pfn = is_vmalloc ? vmalloc_to_pfn(src) : | |
318 | page_to_pfn(virt_to_page(src)); | |
319 | /* | |
320 | * Treat the PTI LDT range as a *userspace* range. | |
321 | * get_locked_pte() will allocate all needed pagetables | |
322 | * and account for them in this mm. | |
323 | */ | |
324 | ptep = get_locked_pte(mm, va, &ptl); | |
325 | if (!ptep) | |
326 | return -ENOMEM; | |
327 | /* | |
328 | * Map it RO so the easy to find address is not a primary | |
329 | * target via some kernel interface which misses a | |
330 | * permission check. | |
331 | */ | |
332 | pte_prot = __pgprot(__PAGE_KERNEL_RO & ~_PAGE_GLOBAL); | |
333 | /* Filter out unsuppored __PAGE_KERNEL* bits: */ | |
334 | pgprot_val(pte_prot) &= __supported_pte_mask; | |
335 | pte = pfn_pte(pfn, pte_prot); | |
336 | set_pte_at(mm, va, ptep, pte); | |
337 | pte_unmap_unlock(ptep, ptl); | |
338 | } | |
339 | ||
340 | /* Propagate LDT mapping to the user page-table */ | |
341 | map_ldt_struct_to_user(mm); | |
342 | ||
343 | ldt->slot = slot; | |
344 | return 0; | |
345 | } | |
346 | ||
347 | static void unmap_ldt_struct(struct mm_struct *mm, struct ldt_struct *ldt) | |
348 | { | |
349 | unsigned long va; | |
350 | int i, nr_pages; | |
351 | ||
352 | if (!ldt) | |
353 | return; | |
354 | ||
355 | /* LDT map/unmap is only required for PTI */ | |
356 | if (!boot_cpu_has(X86_FEATURE_PTI)) | |
357 | return; | |
358 | ||
359 | nr_pages = DIV_ROUND_UP(ldt->nr_entries * LDT_ENTRY_SIZE, PAGE_SIZE); | |
360 | ||
361 | for (i = 0; i < nr_pages; i++) { | |
362 | unsigned long offset = i << PAGE_SHIFT; | |
363 | spinlock_t *ptl; | |
364 | pte_t *ptep; | |
365 | ||
366 | va = (unsigned long)ldt_slot_va(ldt->slot) + offset; | |
367 | ptep = get_locked_pte(mm, va, &ptl); | |
368 | pte_clear(mm, va, ptep); | |
369 | pte_unmap_unlock(ptep, ptl); | |
370 | } | |
371 | ||
372 | va = (unsigned long)ldt_slot_va(ldt->slot); | |
373 | flush_tlb_mm_range(mm, va, va + nr_pages * PAGE_SIZE, PAGE_SHIFT, false); | |
374 | } | |
375 | ||
376 | #else /* !CONFIG_PAGE_TABLE_ISOLATION */ | |
377 | ||
378 | static int | |
379 | map_ldt_struct(struct mm_struct *mm, struct ldt_struct *ldt, int slot) | |
380 | { | |
381 | return 0; | |
382 | } | |
383 | ||
384 | static void unmap_ldt_struct(struct mm_struct *mm, struct ldt_struct *ldt) | |
385 | { | |
386 | } | |
387 | #endif /* CONFIG_PAGE_TABLE_ISOLATION */ | |
388 | ||
389 | static void free_ldt_pgtables(struct mm_struct *mm) | |
390 | { | |
391 | #ifdef CONFIG_PAGE_TABLE_ISOLATION | |
392 | struct mmu_gather tlb; | |
393 | unsigned long start = LDT_BASE_ADDR; | |
394 | unsigned long end = LDT_END_ADDR; | |
395 | ||
396 | if (!boot_cpu_has(X86_FEATURE_PTI)) | |
397 | return; | |
398 | ||
399 | tlb_gather_mmu(&tlb, mm, start, end); | |
400 | free_pgd_range(&tlb, start, end, start, end); | |
401 | tlb_finish_mmu(&tlb, start, end); | |
402 | #endif | |
403 | } | |
404 | ||
405 | /* After calling this, the LDT is immutable. */ | |
406 | static void finalize_ldt_struct(struct ldt_struct *ldt) | |
407 | { | |
408 | paravirt_alloc_ldt(ldt->entries, ldt->nr_entries); | |
409 | } | |
410 | ||
411 | static void install_ldt(struct mm_struct *mm, struct ldt_struct *ldt) | |
412 | { | |
413 | mutex_lock(&mm->context.lock); | |
414 | ||
415 | /* Synchronizes with READ_ONCE in load_mm_ldt. */ | |
416 | smp_store_release(&mm->context.ldt, ldt); | |
417 | ||
418 | /* Activate the LDT for all CPUs using currents mm. */ | |
419 | on_each_cpu_mask(mm_cpumask(mm), flush_ldt, mm, true); | |
420 | ||
421 | mutex_unlock(&mm->context.lock); | |
422 | } | |
423 | ||
424 | static void free_ldt_struct(struct ldt_struct *ldt) | |
425 | { | |
426 | if (likely(!ldt)) | |
427 | return; | |
428 | ||
429 | paravirt_free_ldt(ldt->entries, ldt->nr_entries); | |
430 | if (ldt->nr_entries * LDT_ENTRY_SIZE > PAGE_SIZE) | |
431 | vfree_atomic(ldt->entries); | |
432 | else | |
433 | free_page((unsigned long)ldt->entries); | |
434 | kfree(ldt); | |
435 | } | |
436 | ||
437 | /* | |
438 | * Called on fork from arch_dup_mmap(). Just copy the current LDT state, | |
439 | * the new task is not running, so nothing can be installed. | |
440 | */ | |
441 | int ldt_dup_context(struct mm_struct *old_mm, struct mm_struct *mm) | |
442 | { | |
443 | struct ldt_struct *new_ldt; | |
444 | int retval = 0; | |
445 | ||
446 | if (!old_mm) | |
447 | return 0; | |
448 | ||
449 | mutex_lock(&old_mm->context.lock); | |
450 | if (!old_mm->context.ldt) | |
451 | goto out_unlock; | |
452 | ||
453 | new_ldt = alloc_ldt_struct(old_mm->context.ldt->nr_entries); | |
454 | if (!new_ldt) { | |
455 | retval = -ENOMEM; | |
456 | goto out_unlock; | |
457 | } | |
458 | ||
459 | memcpy(new_ldt->entries, old_mm->context.ldt->entries, | |
460 | new_ldt->nr_entries * LDT_ENTRY_SIZE); | |
461 | finalize_ldt_struct(new_ldt); | |
462 | ||
463 | retval = map_ldt_struct(mm, new_ldt, 0); | |
464 | if (retval) { | |
465 | free_ldt_pgtables(mm); | |
466 | free_ldt_struct(new_ldt); | |
467 | goto out_unlock; | |
468 | } | |
469 | mm->context.ldt = new_ldt; | |
470 | ||
471 | out_unlock: | |
472 | mutex_unlock(&old_mm->context.lock); | |
473 | return retval; | |
474 | } | |
475 | ||
476 | /* | |
477 | * No need to lock the MM as we are the last user | |
478 | * | |
479 | * 64bit: Don't touch the LDT register - we're already in the next thread. | |
480 | */ | |
481 | void destroy_context_ldt(struct mm_struct *mm) | |
482 | { | |
483 | free_ldt_struct(mm->context.ldt); | |
484 | mm->context.ldt = NULL; | |
485 | } | |
486 | ||
487 | void ldt_arch_exit_mmap(struct mm_struct *mm) | |
488 | { | |
489 | free_ldt_pgtables(mm); | |
490 | } | |
491 | ||
492 | static int read_ldt(void __user *ptr, unsigned long bytecount) | |
493 | { | |
494 | struct mm_struct *mm = current->mm; | |
495 | unsigned long entries_size; | |
496 | int retval; | |
497 | ||
498 | down_read(&mm->context.ldt_usr_sem); | |
499 | ||
500 | if (!mm->context.ldt) { | |
501 | retval = 0; | |
502 | goto out_unlock; | |
503 | } | |
504 | ||
505 | if (bytecount > LDT_ENTRY_SIZE * LDT_ENTRIES) | |
506 | bytecount = LDT_ENTRY_SIZE * LDT_ENTRIES; | |
507 | ||
508 | entries_size = mm->context.ldt->nr_entries * LDT_ENTRY_SIZE; | |
509 | if (entries_size > bytecount) | |
510 | entries_size = bytecount; | |
511 | ||
512 | if (copy_to_user(ptr, mm->context.ldt->entries, entries_size)) { | |
513 | retval = -EFAULT; | |
514 | goto out_unlock; | |
515 | } | |
516 | ||
517 | if (entries_size != bytecount) { | |
518 | /* Zero-fill the rest and pretend we read bytecount bytes. */ | |
519 | if (clear_user(ptr + entries_size, bytecount - entries_size)) { | |
520 | retval = -EFAULT; | |
521 | goto out_unlock; | |
522 | } | |
523 | } | |
524 | retval = bytecount; | |
525 | ||
526 | out_unlock: | |
527 | up_read(&mm->context.ldt_usr_sem); | |
528 | return retval; | |
529 | } | |
530 | ||
531 | static int read_default_ldt(void __user *ptr, unsigned long bytecount) | |
532 | { | |
533 | /* CHECKME: Can we use _one_ random number ? */ | |
534 | #ifdef CONFIG_X86_32 | |
535 | unsigned long size = 5 * sizeof(struct desc_struct); | |
536 | #else | |
537 | unsigned long size = 128; | |
538 | #endif | |
539 | if (bytecount > size) | |
540 | bytecount = size; | |
541 | if (clear_user(ptr, bytecount)) | |
542 | return -EFAULT; | |
543 | return bytecount; | |
544 | } | |
545 | ||
546 | static int write_ldt(void __user *ptr, unsigned long bytecount, int oldmode) | |
547 | { | |
548 | struct mm_struct *mm = current->mm; | |
549 | struct ldt_struct *new_ldt, *old_ldt; | |
550 | unsigned int old_nr_entries, new_nr_entries; | |
551 | struct user_desc ldt_info; | |
552 | struct desc_struct ldt; | |
553 | int error; | |
554 | ||
555 | error = -EINVAL; | |
556 | if (bytecount != sizeof(ldt_info)) | |
557 | goto out; | |
558 | error = -EFAULT; | |
559 | if (copy_from_user(&ldt_info, ptr, sizeof(ldt_info))) | |
560 | goto out; | |
561 | ||
562 | error = -EINVAL; | |
563 | if (ldt_info.entry_number >= LDT_ENTRIES) | |
564 | goto out; | |
565 | if (ldt_info.contents == 3) { | |
566 | if (oldmode) | |
567 | goto out; | |
568 | if (ldt_info.seg_not_present == 0) | |
569 | goto out; | |
570 | } | |
571 | ||
572 | if ((oldmode && !ldt_info.base_addr && !ldt_info.limit) || | |
573 | LDT_empty(&ldt_info)) { | |
574 | /* The user wants to clear the entry. */ | |
575 | memset(&ldt, 0, sizeof(ldt)); | |
576 | } else { | |
577 | if (!IS_ENABLED(CONFIG_X86_16BIT) && !ldt_info.seg_32bit) { | |
578 | error = -EINVAL; | |
579 | goto out; | |
580 | } | |
581 | ||
582 | fill_ldt(&ldt, &ldt_info); | |
583 | if (oldmode) | |
584 | ldt.avl = 0; | |
585 | } | |
586 | ||
587 | if (down_write_killable(&mm->context.ldt_usr_sem)) | |
588 | return -EINTR; | |
589 | ||
590 | old_ldt = mm->context.ldt; | |
591 | old_nr_entries = old_ldt ? old_ldt->nr_entries : 0; | |
592 | new_nr_entries = max(ldt_info.entry_number + 1, old_nr_entries); | |
593 | ||
594 | error = -ENOMEM; | |
595 | new_ldt = alloc_ldt_struct(new_nr_entries); | |
596 | if (!new_ldt) | |
597 | goto out_unlock; | |
598 | ||
599 | if (old_ldt) | |
600 | memcpy(new_ldt->entries, old_ldt->entries, old_nr_entries * LDT_ENTRY_SIZE); | |
601 | ||
602 | new_ldt->entries[ldt_info.entry_number] = ldt; | |
603 | finalize_ldt_struct(new_ldt); | |
604 | ||
605 | /* | |
606 | * If we are using PTI, map the new LDT into the userspace pagetables. | |
607 | * If there is already an LDT, use the other slot so that other CPUs | |
608 | * will continue to use the old LDT until install_ldt() switches | |
609 | * them over to the new LDT. | |
610 | */ | |
611 | error = map_ldt_struct(mm, new_ldt, old_ldt ? !old_ldt->slot : 0); | |
612 | if (error) { | |
613 | /* | |
614 | * This only can fail for the first LDT setup. If an LDT is | |
615 | * already installed then the PTE page is already | |
616 | * populated. Mop up a half populated page table. | |
617 | */ | |
618 | if (!WARN_ON_ONCE(old_ldt)) | |
619 | free_ldt_pgtables(mm); | |
620 | free_ldt_struct(new_ldt); | |
621 | goto out_unlock; | |
622 | } | |
623 | ||
624 | install_ldt(mm, new_ldt); | |
625 | unmap_ldt_struct(mm, old_ldt); | |
626 | free_ldt_struct(old_ldt); | |
627 | error = 0; | |
628 | ||
629 | out_unlock: | |
630 | up_write(&mm->context.ldt_usr_sem); | |
631 | out: | |
632 | return error; | |
633 | } | |
634 | ||
635 | SYSCALL_DEFINE3(modify_ldt, int , func , void __user * , ptr , | |
636 | unsigned long , bytecount) | |
637 | { | |
638 | int ret = -ENOSYS; | |
639 | ||
640 | switch (func) { | |
641 | case 0: | |
642 | ret = read_ldt(ptr, bytecount); | |
643 | break; | |
644 | case 1: | |
645 | ret = write_ldt(ptr, bytecount, 1); | |
646 | break; | |
647 | case 2: | |
648 | ret = read_default_ldt(ptr, bytecount); | |
649 | break; | |
650 | case 0x11: | |
651 | ret = write_ldt(ptr, bytecount, 0); | |
652 | break; | |
653 | } | |
654 | /* | |
655 | * The SYSCALL_DEFINE() macros give us an 'unsigned long' | |
656 | * return type, but tht ABI for sys_modify_ldt() expects | |
657 | * 'int'. This cast gives us an int-sized value in %rax | |
658 | * for the return code. The 'unsigned' is necessary so | |
659 | * the compiler does not try to sign-extend the negative | |
660 | * return codes into the high half of the register when | |
661 | * taking the value from int->long. | |
662 | */ | |
663 | return (unsigned int)ret; | |
664 | } |