]>
git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - arch/x86/mm/mpx.c
aad4ac386f98c34d7cec84a35974b22207bcea6a
2 * mpx.c - Memory Protection eXtensions
4 * Copyright (c) 2014, Intel Corporation.
5 * Qiaowei Ren <qiaowei.ren@intel.com>
6 * Dave Hansen <dave.hansen@intel.com>
8 #include <linux/kernel.h>
9 #include <linux/slab.h>
10 #include <linux/syscalls.h>
11 #include <linux/sched/sysctl.h>
15 #include <asm/mmu_context.h>
17 #include <asm/processor.h>
18 #include <asm/fpu/internal.h>
20 #define CREATE_TRACE_POINTS
21 #include <asm/trace/mpx.h>
23 static inline unsigned long mpx_bd_size_bytes(struct mm_struct
*mm
)
26 return MPX_BD_SIZE_BYTES_64
;
28 return MPX_BD_SIZE_BYTES_32
;
31 static inline unsigned long mpx_bt_size_bytes(struct mm_struct
*mm
)
34 return MPX_BT_SIZE_BYTES_64
;
36 return MPX_BT_SIZE_BYTES_32
;
40 * This is really a simplified "vm_mmap". it only handles MPX
41 * bounds tables (the bounds directory is user-allocated).
43 static unsigned long mpx_mmap(unsigned long len
)
45 struct mm_struct
*mm
= current
->mm
;
46 unsigned long addr
, populate
;
48 /* Only bounds table can be allocated here */
49 if (len
!= mpx_bt_size_bytes(mm
))
52 down_write(&mm
->mmap_sem
);
53 addr
= do_mmap(NULL
, 0, len
, PROT_READ
| PROT_WRITE
,
54 MAP_ANONYMOUS
| MAP_PRIVATE
, VM_MPX
, 0, &populate
);
55 up_write(&mm
->mmap_sem
);
57 mm_populate(addr
, populate
);
68 static int get_reg_offset(struct insn
*insn
, struct pt_regs
*regs
,
73 static const int regoff
[] = {
74 offsetof(struct pt_regs
, ax
),
75 offsetof(struct pt_regs
, cx
),
76 offsetof(struct pt_regs
, dx
),
77 offsetof(struct pt_regs
, bx
),
78 offsetof(struct pt_regs
, sp
),
79 offsetof(struct pt_regs
, bp
),
80 offsetof(struct pt_regs
, si
),
81 offsetof(struct pt_regs
, di
),
83 offsetof(struct pt_regs
, r8
),
84 offsetof(struct pt_regs
, r9
),
85 offsetof(struct pt_regs
, r10
),
86 offsetof(struct pt_regs
, r11
),
87 offsetof(struct pt_regs
, r12
),
88 offsetof(struct pt_regs
, r13
),
89 offsetof(struct pt_regs
, r14
),
90 offsetof(struct pt_regs
, r15
),
93 int nr_registers
= ARRAY_SIZE(regoff
);
95 * Don't possibly decode a 32-bit instructions as
96 * reading a 64-bit-only register.
98 if (IS_ENABLED(CONFIG_X86_64
) && !insn
->x86_64
)
103 regno
= X86_MODRM_RM(insn
->modrm
.value
);
104 if (X86_REX_B(insn
->rex_prefix
.value
))
109 regno
= X86_SIB_INDEX(insn
->sib
.value
);
110 if (X86_REX_X(insn
->rex_prefix
.value
))
115 regno
= X86_SIB_BASE(insn
->sib
.value
);
116 if (X86_REX_B(insn
->rex_prefix
.value
))
121 pr_err("invalid register type");
126 if (regno
>= nr_registers
) {
127 WARN_ONCE(1, "decoded an instruction with an invalid register");
130 return regoff
[regno
];
134 * return the address being referenced be instruction
135 * for rm=3 returning the content of the rm reg
136 * for rm!=3 calculates the address using SIB and Disp
138 static void __user
*mpx_get_addr_ref(struct insn
*insn
, struct pt_regs
*regs
)
140 unsigned long addr
, base
, indx
;
141 int addr_offset
, base_offset
, indx_offset
;
144 insn_get_modrm(insn
);
146 sib
= insn
->sib
.value
;
148 if (X86_MODRM_MOD(insn
->modrm
.value
) == 3) {
149 addr_offset
= get_reg_offset(insn
, regs
, REG_TYPE_RM
);
152 addr
= regs_get_register(regs
, addr_offset
);
154 if (insn
->sib
.nbytes
) {
155 base_offset
= get_reg_offset(insn
, regs
, REG_TYPE_BASE
);
159 indx_offset
= get_reg_offset(insn
, regs
, REG_TYPE_INDEX
);
163 base
= regs_get_register(regs
, base_offset
);
164 indx
= regs_get_register(regs
, indx_offset
);
165 addr
= base
+ indx
* (1 << X86_SIB_SCALE(sib
));
167 addr_offset
= get_reg_offset(insn
, regs
, REG_TYPE_RM
);
170 addr
= regs_get_register(regs
, addr_offset
);
172 addr
+= insn
->displacement
.value
;
174 return (void __user
*)addr
;
176 return (void __user
*)-1;
179 static int mpx_insn_decode(struct insn
*insn
,
180 struct pt_regs
*regs
)
182 unsigned char buf
[MAX_INSN_SIZE
];
183 int x86_64
= !test_thread_flag(TIF_IA32
);
187 not_copied
= copy_from_user(buf
, (void __user
*)regs
->ip
, sizeof(buf
));
188 nr_copied
= sizeof(buf
) - not_copied
;
190 * The decoder _should_ fail nicely if we pass it a short buffer.
191 * But, let's not depend on that implementation detail. If we
192 * did not get anything, just error out now.
196 insn_init(insn
, buf
, nr_copied
, x86_64
);
197 insn_get_length(insn
);
199 * copy_from_user() tries to get as many bytes as we could see in
200 * the largest possible instruction. If the instruction we are
201 * after is shorter than that _and_ we attempt to copy from
202 * something unreadable, we might get a short read. This is OK
203 * as long as the read did not stop in the middle of the
204 * instruction. Check to see if we got a partial instruction.
206 if (nr_copied
< insn
->length
)
209 insn_get_opcode(insn
);
211 * We only _really_ need to decode bndcl/bndcn/bndcu
212 * Error out on anything else.
214 if (insn
->opcode
.bytes
[0] != 0x0f)
216 if ((insn
->opcode
.bytes
[1] != 0x1a) &&
217 (insn
->opcode
.bytes
[1] != 0x1b))
226 * If a bounds overflow occurs then a #BR is generated. This
227 * function decodes MPX instructions to get violation address
228 * and set this address into extended struct siginfo.
230 * Note that this is not a super precise way of doing this.
231 * Userspace could have, by the time we get here, written
232 * anything it wants in to the instructions. We can not
233 * trust anything about it. They might not be valid
234 * instructions or might encode invalid registers, etc...
236 * The caller is expected to kfree() the returned siginfo_t.
238 siginfo_t
*mpx_generate_siginfo(struct pt_regs
*regs
)
240 const struct mpx_bndreg_state
*bndregs
;
241 const struct mpx_bndreg
*bndreg
;
242 siginfo_t
*info
= NULL
;
247 err
= mpx_insn_decode(&insn
, regs
);
252 * We know at this point that we are only dealing with
255 insn_get_modrm(&insn
);
256 bndregno
= X86_MODRM_REG(insn
.modrm
.value
);
261 /* get bndregs field from current task's xsave area */
262 bndregs
= get_xsave_field_ptr(XFEATURE_MASK_BNDREGS
);
267 /* now go select the individual register in the set of 4 */
268 bndreg
= &bndregs
->bndreg
[bndregno
];
270 info
= kzalloc(sizeof(*info
), GFP_KERNEL
);
276 * The registers are always 64-bit, but the upper 32
277 * bits are ignored in 32-bit mode. Also, note that the
278 * upper bounds are architecturally represented in 1's
281 * The 'unsigned long' cast is because the compiler
282 * complains when casting from integers to different-size
285 info
->si_lower
= (void __user
*)(unsigned long)bndreg
->lower_bound
;
286 info
->si_upper
= (void __user
*)(unsigned long)~bndreg
->upper_bound
;
287 info
->si_addr_lsb
= 0;
288 info
->si_signo
= SIGSEGV
;
290 info
->si_code
= SEGV_BNDERR
;
291 info
->si_addr
= mpx_get_addr_ref(&insn
, regs
);
293 * We were not able to extract an address from the instruction,
294 * probably because there was something invalid in it.
296 if (info
->si_addr
== (void __user
*)-1) {
300 trace_mpx_bounds_register_exception(info
->si_addr
, bndreg
);
303 /* info might be NULL, but kfree() handles that */
308 static __user
void *mpx_get_bounds_dir(void)
310 const struct mpx_bndcsr
*bndcsr
;
312 if (!cpu_feature_enabled(X86_FEATURE_MPX
))
313 return MPX_INVALID_BOUNDS_DIR
;
316 * The bounds directory pointer is stored in a register
317 * only accessible if we first do an xsave.
319 bndcsr
= get_xsave_field_ptr(XFEATURE_MASK_BNDCSR
);
321 return MPX_INVALID_BOUNDS_DIR
;
324 * Make sure the register looks valid by checking the
327 if (!(bndcsr
->bndcfgu
& MPX_BNDCFG_ENABLE_FLAG
))
328 return MPX_INVALID_BOUNDS_DIR
;
331 * Lastly, mask off the low bits used for configuration
332 * flags, and return the address of the bounds table.
334 return (void __user
*)(unsigned long)
335 (bndcsr
->bndcfgu
& MPX_BNDCFG_ADDR_MASK
);
338 int mpx_enable_management(void)
340 void __user
*bd_base
= MPX_INVALID_BOUNDS_DIR
;
341 struct mm_struct
*mm
= current
->mm
;
345 * runtime in the userspace will be responsible for allocation of
346 * the bounds directory. Then, it will save the base of the bounds
347 * directory into XSAVE/XRSTOR Save Area and enable MPX through
348 * XRSTOR instruction.
350 * The copy_xregs_to_kernel() beneath get_xsave_field_ptr() is
351 * expected to be relatively expensive. Storing the bounds
352 * directory here means that we do not have to do xsave in the
353 * unmap path; we can just use mm->context.bd_addr instead.
355 bd_base
= mpx_get_bounds_dir();
356 down_write(&mm
->mmap_sem
);
357 mm
->context
.bd_addr
= bd_base
;
358 if (mm
->context
.bd_addr
== MPX_INVALID_BOUNDS_DIR
)
361 up_write(&mm
->mmap_sem
);
365 int mpx_disable_management(void)
367 struct mm_struct
*mm
= current
->mm
;
369 if (!cpu_feature_enabled(X86_FEATURE_MPX
))
372 down_write(&mm
->mmap_sem
);
373 mm
->context
.bd_addr
= MPX_INVALID_BOUNDS_DIR
;
374 up_write(&mm
->mmap_sem
);
378 static int mpx_cmpxchg_bd_entry(struct mm_struct
*mm
,
379 unsigned long *curval
,
380 unsigned long __user
*addr
,
381 unsigned long old_val
, unsigned long new_val
)
385 * user_atomic_cmpxchg_inatomic() actually uses sizeof()
386 * the pointer that we pass to it to figure out how much
387 * data to cmpxchg. We have to be careful here not to
388 * pass a pointer to a 64-bit data type when we only want
391 if (is_64bit_mm(mm
)) {
392 ret
= user_atomic_cmpxchg_inatomic(curval
,
393 addr
, old_val
, new_val
);
395 u32
uninitialized_var(curval_32
);
396 u32 old_val_32
= old_val
;
397 u32 new_val_32
= new_val
;
398 u32 __user
*addr_32
= (u32 __user
*)addr
;
400 ret
= user_atomic_cmpxchg_inatomic(&curval_32
,
401 addr_32
, old_val_32
, new_val_32
);
408 * With 32-bit mode, a bounds directory is 4MB, and the size of each
409 * bounds table is 16KB. With 64-bit mode, a bounds directory is 2GB,
410 * and the size of each bounds table is 4MB.
412 static int allocate_bt(struct mm_struct
*mm
, long __user
*bd_entry
)
414 unsigned long expected_old_val
= 0;
415 unsigned long actual_old_val
= 0;
416 unsigned long bt_addr
;
417 unsigned long bd_new_entry
;
421 * Carve the virtual space out of userspace for the new
424 bt_addr
= mpx_mmap(mpx_bt_size_bytes(mm
));
425 if (IS_ERR((void *)bt_addr
))
426 return PTR_ERR((void *)bt_addr
);
428 * Set the valid flag (kinda like _PAGE_PRESENT in a pte)
430 bd_new_entry
= bt_addr
| MPX_BD_ENTRY_VALID_FLAG
;
433 * Go poke the address of the new bounds table in to the
434 * bounds directory entry out in userspace memory. Note:
435 * we may race with another CPU instantiating the same table.
436 * In that case the cmpxchg will see an unexpected
439 * This can fault, but that's OK because we do not hold
440 * mmap_sem at this point, unlike some of the other part
441 * of the MPX code that have to pagefault_disable().
443 ret
= mpx_cmpxchg_bd_entry(mm
, &actual_old_val
, bd_entry
,
444 expected_old_val
, bd_new_entry
);
449 * The user_atomic_cmpxchg_inatomic() will only return nonzero
450 * for faults, *not* if the cmpxchg itself fails. Now we must
451 * verify that the cmpxchg itself completed successfully.
454 * We expected an empty 'expected_old_val', but instead found
455 * an apparently valid entry. Assume we raced with another
456 * thread to instantiate this table and desclare succecss.
458 if (actual_old_val
& MPX_BD_ENTRY_VALID_FLAG
) {
463 * We found a non-empty bd_entry but it did not have the
464 * VALID_FLAG set. Return an error which will result in
465 * a SEGV since this probably means that somebody scribbled
466 * some invalid data in to a bounds table.
468 if (expected_old_val
!= actual_old_val
) {
472 trace_mpx_new_bounds_table(bt_addr
);
475 vm_munmap(bt_addr
, mpx_bt_size_bytes(mm
));
480 * When a BNDSTX instruction attempts to save bounds to a bounds
481 * table, it will first attempt to look up the table in the
482 * first-level bounds directory. If it does not find a table in
483 * the directory, a #BR is generated and we get here in order to
484 * allocate a new table.
486 * With 32-bit mode, the size of BD is 4MB, and the size of each
487 * bound table is 16KB. With 64-bit mode, the size of BD is 2GB,
488 * and the size of each bound table is 4MB.
490 static int do_mpx_bt_fault(void)
492 unsigned long bd_entry
, bd_base
;
493 const struct mpx_bndcsr
*bndcsr
;
494 struct mm_struct
*mm
= current
->mm
;
496 bndcsr
= get_xsave_field_ptr(XFEATURE_MASK_BNDCSR
);
500 * Mask off the preserve and enable bits
502 bd_base
= bndcsr
->bndcfgu
& MPX_BNDCFG_ADDR_MASK
;
504 * The hardware provides the address of the missing or invalid
505 * entry via BNDSTATUS, so we don't have to go look it up.
507 bd_entry
= bndcsr
->bndstatus
& MPX_BNDSTA_ADDR_MASK
;
509 * Make sure the directory entry is within where we think
512 if ((bd_entry
< bd_base
) ||
513 (bd_entry
>= bd_base
+ mpx_bd_size_bytes(mm
)))
516 return allocate_bt(mm
, (long __user
*)bd_entry
);
519 int mpx_handle_bd_fault(void)
522 * Userspace never asked us to manage the bounds tables,
525 if (!kernel_managing_mpx_tables(current
->mm
))
528 if (do_mpx_bt_fault()) {
529 force_sig(SIGSEGV
, current
);
531 * The force_sig() is essentially "handling" this
532 * exception, so we do not pass up the error
533 * from do_mpx_bt_fault().
540 * A thin wrapper around get_user_pages(). Returns 0 if the
541 * fault was resolved or -errno if not.
543 static int mpx_resolve_fault(long __user
*addr
, int write
)
548 gup_ret
= get_user_pages((unsigned long)addr
, nr_pages
,
549 write
? FOLL_WRITE
: 0, NULL
, NULL
);
551 * get_user_pages() returns number of pages gotten.
552 * 0 means we failed to fault in and get anything,
553 * probably because 'addr' is bad.
557 /* Other error, return it */
560 /* must have gup'd a page and gup_ret>0, success */
564 static unsigned long mpx_bd_entry_to_bt_addr(struct mm_struct
*mm
,
565 unsigned long bd_entry
)
567 unsigned long bt_addr
= bd_entry
;
570 * Bit 0 in a bt_entry is always the valid bit.
572 bt_addr
&= ~MPX_BD_ENTRY_VALID_FLAG
;
574 * Tables are naturally aligned at 8-byte boundaries
575 * on 64-bit and 4-byte boundaries on 32-bit. The
576 * documentation makes it appear that the low bits
577 * are ignored by the hardware, so we do the same.
583 bt_addr
&= ~(align_to_bytes
-1);
588 * We only want to do a 4-byte get_user() on 32-bit. Otherwise,
589 * we might run off the end of the bounds table if we are on
590 * a 64-bit kernel and try to get 8 bytes.
592 int get_user_bd_entry(struct mm_struct
*mm
, unsigned long *bd_entry_ret
,
593 long __user
*bd_entry_ptr
)
599 return get_user(*bd_entry_ret
, bd_entry_ptr
);
602 * Note that get_user() uses the type of the *pointer* to
603 * establish the size of the get, not the destination.
605 ret
= get_user(bd_entry_32
, (u32 __user
*)bd_entry_ptr
);
606 *bd_entry_ret
= bd_entry_32
;
611 * Get the base of bounds tables pointed by specific bounds
614 static int get_bt_addr(struct mm_struct
*mm
,
615 long __user
*bd_entry_ptr
,
616 unsigned long *bt_addr_result
)
620 unsigned long bd_entry
;
621 unsigned long bt_addr
;
623 if (!access_ok(VERIFY_READ
, (bd_entry_ptr
), sizeof(*bd_entry_ptr
)))
630 ret
= get_user_bd_entry(mm
, &bd_entry
, bd_entry_ptr
);
635 ret
= mpx_resolve_fault(bd_entry_ptr
, need_write
);
637 * If we could not resolve the fault, consider it
638 * userspace's fault and error out.
644 valid_bit
= bd_entry
& MPX_BD_ENTRY_VALID_FLAG
;
645 bt_addr
= mpx_bd_entry_to_bt_addr(mm
, bd_entry
);
648 * When the kernel is managing bounds tables, a bounds directory
649 * entry will either have a valid address (plus the valid bit)
650 * *OR* be completely empty. If we see a !valid entry *and* some
651 * data in the address field, we know something is wrong. This
652 * -EINVAL return will cause a SIGSEGV.
654 if (!valid_bit
&& bt_addr
)
657 * Do we have an completely zeroed bt entry? That is OK. It
658 * just means there was no bounds table for this memory. Make
659 * sure to distinguish this from -EINVAL, which will cause
665 *bt_addr_result
= bt_addr
;
669 static inline int bt_entry_size_bytes(struct mm_struct
*mm
)
672 return MPX_BT_ENTRY_BYTES_64
;
674 return MPX_BT_ENTRY_BYTES_32
;
678 * Take a virtual address and turns it in to the offset in bytes
679 * inside of the bounds table where the bounds table entry
680 * controlling 'addr' can be found.
682 static unsigned long mpx_get_bt_entry_offset_bytes(struct mm_struct
*mm
,
685 unsigned long bt_table_nr_entries
;
686 unsigned long offset
= addr
;
688 if (is_64bit_mm(mm
)) {
689 /* Bottom 3 bits are ignored on 64-bit */
691 bt_table_nr_entries
= MPX_BT_NR_ENTRIES_64
;
693 /* Bottom 2 bits are ignored on 32-bit */
695 bt_table_nr_entries
= MPX_BT_NR_ENTRIES_32
;
698 * We know the size of the table in to which we are
699 * indexing, and we have eliminated all the low bits
700 * which are ignored for indexing.
702 * Mask out all the high bits which we do not need
703 * to index in to the table. Note that the tables
704 * are always powers of two so this gives us a proper
707 offset
&= (bt_table_nr_entries
-1);
709 * We now have an entry offset in terms of *entries* in
710 * the table. We need to scale it back up to bytes.
712 offset
*= bt_entry_size_bytes(mm
);
717 * How much virtual address space does a single bounds
718 * directory entry cover?
720 * Note, we need a long long because 4GB doesn't fit in
721 * to a long on 32-bit.
723 static inline unsigned long bd_entry_virt_space(struct mm_struct
*mm
)
725 unsigned long long virt_space
;
726 unsigned long long GB
= (1ULL << 30);
729 * This covers 32-bit emulation as well as 32-bit kernels
730 * running on 64-bit hardware.
732 if (!is_64bit_mm(mm
))
733 return (4ULL * GB
) / MPX_BD_NR_ENTRIES_32
;
736 * 'x86_virt_bits' returns what the hardware is capable
737 * of, and returns the full >32-bit address space when
738 * running 32-bit kernels on 64-bit hardware.
740 virt_space
= (1ULL << boot_cpu_data
.x86_virt_bits
);
741 return virt_space
/ MPX_BD_NR_ENTRIES_64
;
745 * Free the backing physical pages of bounds table 'bt_addr'.
746 * Assume start...end is within that bounds table.
748 static noinline
int zap_bt_entries_mapping(struct mm_struct
*mm
,
749 unsigned long bt_addr
,
750 unsigned long start_mapping
, unsigned long end_mapping
)
752 struct vm_area_struct
*vma
;
753 unsigned long addr
, len
;
758 * if we 'end' on a boundary, the offset will be 0 which
759 * is not what we want. Back it up a byte to get the
760 * last bt entry. Then once we have the entry itself,
761 * move 'end' back up by the table entry size.
763 start
= bt_addr
+ mpx_get_bt_entry_offset_bytes(mm
, start_mapping
);
764 end
= bt_addr
+ mpx_get_bt_entry_offset_bytes(mm
, end_mapping
- 1);
766 * Move end back up by one entry. Among other things
767 * this ensures that it remains page-aligned and does
768 * not screw up zap_page_range()
770 end
+= bt_entry_size_bytes(mm
);
773 * Find the first overlapping vma. If vma->vm_start > start, there
774 * will be a hole in the bounds table. This -EINVAL return will
777 vma
= find_vma(mm
, start
);
778 if (!vma
|| vma
->vm_start
> start
)
782 * A NUMA policy on a VM_MPX VMA could cause this bounds table to
783 * be split. So we need to look across the entire 'start -> end'
784 * range of this bounds table, find all of the VM_MPX VMAs, and
788 while (vma
&& vma
->vm_start
< end
) {
790 * We followed a bounds directory entry down
791 * here. If we find a non-MPX VMA, that's bad,
792 * so stop immediately and return an error. This
793 * probably results in a SIGSEGV.
795 if (!(vma
->vm_flags
& VM_MPX
))
798 len
= min(vma
->vm_end
, end
) - addr
;
799 zap_page_range(vma
, addr
, len
);
800 trace_mpx_unmap_zap(addr
, addr
+len
);
803 addr
= vma
->vm_start
;
808 static unsigned long mpx_get_bd_entry_offset(struct mm_struct
*mm
,
812 * There are several ways to derive the bd offsets. We
813 * use the following approach here:
814 * 1. We know the size of the virtual address space
815 * 2. We know the number of entries in a bounds table
816 * 3. We know that each entry covers a fixed amount of
817 * virtual address space.
818 * So, we can just divide the virtual address by the
819 * virtual space used by one entry to determine which
820 * entry "controls" the given virtual address.
822 if (is_64bit_mm(mm
)) {
823 int bd_entry_size
= 8; /* 64-bit pointer */
825 * Take the 64-bit addressing hole in to account.
827 addr
&= ((1UL << boot_cpu_data
.x86_virt_bits
) - 1);
828 return (addr
/ bd_entry_virt_space(mm
)) * bd_entry_size
;
830 int bd_entry_size
= 4; /* 32-bit pointer */
832 * 32-bit has no hole so this case needs no mask
834 return (addr
/ bd_entry_virt_space(mm
)) * bd_entry_size
;
837 * The two return calls above are exact copies. If we
838 * pull out a single copy and put it in here, gcc won't
839 * realize that we're doing a power-of-2 divide and use
840 * shifts. It uses a real divide. If we put them up
841 * there, it manages to figure it out (gcc 4.8.3).
845 static int unmap_entire_bt(struct mm_struct
*mm
,
846 long __user
*bd_entry
, unsigned long bt_addr
)
848 unsigned long expected_old_val
= bt_addr
| MPX_BD_ENTRY_VALID_FLAG
;
849 unsigned long uninitialized_var(actual_old_val
);
854 unsigned long cleared_bd_entry
= 0;
857 ret
= mpx_cmpxchg_bd_entry(mm
, &actual_old_val
,
858 bd_entry
, expected_old_val
, cleared_bd_entry
);
863 ret
= mpx_resolve_fault(bd_entry
, need_write
);
865 * If we could not resolve the fault, consider it
866 * userspace's fault and error out.
872 * The cmpxchg was performed, check the results.
874 if (actual_old_val
!= expected_old_val
) {
876 * Someone else raced with us to unmap the table.
877 * That is OK, since we were both trying to do
878 * the same thing. Declare success.
883 * Something messed with the bounds directory
884 * entry. We hold mmap_sem for read or write
885 * here, so it could not be a _new_ bounds table
886 * that someone just allocated. Something is
887 * wrong, so pass up the error and SIGSEGV.
892 * Note, we are likely being called under do_munmap() already. To
893 * avoid recursion, do_munmap() will check whether it comes
894 * from one bounds table through VM_MPX flag.
896 return do_munmap(mm
, bt_addr
, mpx_bt_size_bytes(mm
));
899 static int try_unmap_single_bt(struct mm_struct
*mm
,
900 unsigned long start
, unsigned long end
)
902 struct vm_area_struct
*next
;
903 struct vm_area_struct
*prev
;
905 * "bta" == Bounds Table Area: the area controlled by the
906 * bounds table that we are unmapping.
908 unsigned long bta_start_vaddr
= start
& ~(bd_entry_virt_space(mm
)-1);
909 unsigned long bta_end_vaddr
= bta_start_vaddr
+ bd_entry_virt_space(mm
);
910 unsigned long uninitialized_var(bt_addr
);
911 void __user
*bde_vaddr
;
914 * We already unlinked the VMAs from the mm's rbtree so 'start'
915 * is guaranteed to be in a hole. This gets us the first VMA
916 * before the hole in to 'prev' and the next VMA after the hole
919 next
= find_vma_prev(mm
, start
, &prev
);
921 * Do not count other MPX bounds table VMAs as neighbors.
922 * Although theoretically possible, we do not allow bounds
923 * tables for bounds tables so our heads do not explode.
924 * If we count them as neighbors here, we may end up with
925 * lots of tables even though we have no actual table
928 while (next
&& (next
->vm_flags
& VM_MPX
))
929 next
= next
->vm_next
;
930 while (prev
&& (prev
->vm_flags
& VM_MPX
))
931 prev
= prev
->vm_prev
;
933 * We know 'start' and 'end' lie within an area controlled
934 * by a single bounds table. See if there are any other
935 * VMAs controlled by that bounds table. If there are not
936 * then we can "expand" the are we are unmapping to possibly
937 * cover the entire table.
939 next
= find_vma_prev(mm
, start
, &prev
);
940 if ((!prev
|| prev
->vm_end
<= bta_start_vaddr
) &&
941 (!next
|| next
->vm_start
>= bta_end_vaddr
)) {
943 * No neighbor VMAs controlled by same bounds
944 * table. Try to unmap the whole thing
946 start
= bta_start_vaddr
;
950 bde_vaddr
= mm
->context
.bd_addr
+ mpx_get_bd_entry_offset(mm
, start
);
951 ret
= get_bt_addr(mm
, bde_vaddr
, &bt_addr
);
953 * No bounds table there, so nothing to unmap.
955 if (ret
== -ENOENT
) {
962 * We are unmapping an entire table. Either because the
963 * unmap that started this whole process was large enough
964 * to cover an entire table, or that the unmap was small
965 * but was the area covered by a bounds table.
967 if ((start
== bta_start_vaddr
) &&
968 (end
== bta_end_vaddr
))
969 return unmap_entire_bt(mm
, bde_vaddr
, bt_addr
);
970 return zap_bt_entries_mapping(mm
, bt_addr
, start
, end
);
973 static int mpx_unmap_tables(struct mm_struct
*mm
,
974 unsigned long start
, unsigned long end
)
976 unsigned long one_unmap_start
;
977 trace_mpx_unmap_search(start
, end
);
979 one_unmap_start
= start
;
980 while (one_unmap_start
< end
) {
982 unsigned long next_unmap_start
= ALIGN(one_unmap_start
+1,
983 bd_entry_virt_space(mm
));
984 unsigned long one_unmap_end
= end
;
986 * if the end is beyond the current bounds table,
987 * move it back so we only deal with a single one
990 if (one_unmap_end
> next_unmap_start
)
991 one_unmap_end
= next_unmap_start
;
992 ret
= try_unmap_single_bt(mm
, one_unmap_start
, one_unmap_end
);
996 one_unmap_start
= next_unmap_start
;
1002 * Free unused bounds tables covered in a virtual address region being
1003 * munmap()ed. Assume end > start.
1005 * This function will be called by do_munmap(), and the VMAs covering
1006 * the virtual address region start...end have already been split if
1007 * necessary, and the 'vma' is the first vma in this range (start -> end).
1009 void mpx_notify_unmap(struct mm_struct
*mm
, struct vm_area_struct
*vma
,
1010 unsigned long start
, unsigned long end
)
1015 * Refuse to do anything unless userspace has asked
1016 * the kernel to help manage the bounds tables,
1018 if (!kernel_managing_mpx_tables(current
->mm
))
1021 * This will look across the entire 'start -> end' range,
1022 * and find all of the non-VM_MPX VMAs.
1024 * To avoid recursion, if a VM_MPX vma is found in the range
1025 * (start->end), we will not continue follow-up work. This
1026 * recursion represents having bounds tables for bounds tables,
1027 * which should not occur normally. Being strict about it here
1028 * helps ensure that we do not have an exploitable stack overflow.
1031 if (vma
->vm_flags
& VM_MPX
)
1034 } while (vma
&& vma
->vm_start
< end
);
1036 ret
= mpx_unmap_tables(mm
, start
, end
);
1038 force_sig(SIGSEGV
, current
);