2 * mpx-mini-test.c: routines to test Intel MPX (Memory Protection eXtentions)
5 * "Ren, Qiaowei" <qiaowei.ren@intel.com>
6 * "Wei, Gang" <gang.wei@intel.com>
7 * "Hansen, Dave" <dave.hansen@intel.com>
9 * This program is free software; you can redistribute it and/or modify it
10 * under the terms and conditions of the GNU General Public License,
15 * 2014-12-05: Dave Hansen: fixed all of the compiler warnings, and made sure
19 int inspect_every_this_many_mallocs
= 100;
20 int zap_all_every_this_many_mallocs
= 1000;
23 #define _LARGEFILE64_SOURCE
34 #include <sys/types.h>
40 #include "mpx-debug.h"
43 #ifndef __always_inline
44 #define __always_inline inline __attribute__((always_inline)
47 #ifndef TEST_DURATION_SECS
48 #define TEST_DURATION_SECS 3
51 void write_int_to(char *prefix
, char *file
, int int_to_write
)
54 int fd
= open(file
, O_RDWR
);
59 len
= snprintf(buf
, sizeof(buf
), "%s%d", prefix
, int_to_write
);
61 assert(len
< sizeof(buf
));
62 ret
= write(fd
, buf
, len
);
68 void write_pid_to(char *prefix
, char *file
)
70 write_int_to(prefix
, file
, getpid());
75 /* tracing events dir */
76 #define TED "/sys/kernel/debug/tracing/events/"
78 write_pid_to("common_pid=", TED "signal/filter");
79 write_pid_to("common_pid=", TED "exceptions/filter");
80 write_int_to("", TED "signal/enable", 1);
81 write_int_to("", TED "exceptions/enable", 1);
83 write_pid_to("", "/sys/kernel/debug/tracing/set_ftrace_pid");
84 write_int_to("", "/sys/kernel/debug/tracing/trace", 0);
87 #define test_failed() __test_failed(__FILE__, __LINE__)
88 static void __test_failed(char *f
, int l
)
90 fprintf(stderr
, "abort @ %s::%d\n", f
, l
);
95 #define eprintf(args...) fprintf(stderr, args)
99 /* i386 directory size is 4MB */
100 #define REG_IP_IDX REG_EIP
103 #define XSAVE_OFFSET_IN_FPMEM sizeof(struct _libc_fpstate)
106 * __cpuid() is from the Linux Kernel:
108 static inline void __cpuid(unsigned int *eax
, unsigned int *ebx
,
109 unsigned int *ecx
, unsigned int *edx
)
111 /* ecx is often an input as well as an output. */
121 : "0" (*eax
), "2" (*ecx
));
126 #define REG_IP_IDX REG_RIP
127 #define REX_PREFIX "0x48, "
129 #define XSAVE_OFFSET_IN_FPMEM 0
132 * __cpuid() is from the Linux Kernel:
134 static inline void __cpuid(unsigned int *eax
, unsigned int *ebx
,
135 unsigned int *ecx
, unsigned int *edx
)
137 /* ecx is often an input as well as an output. */
144 : "0" (*eax
), "2" (*ecx
));
147 #endif /* !__i386__ */
149 struct xsave_hdr_struct
{
151 uint64_t reserved1
[2];
152 uint64_t reserved2
[5];
153 } __attribute__((packed
));
155 struct bndregs_struct
{
157 } __attribute__((packed
));
159 struct bndcsr_struct
{
162 } __attribute__((packed
));
164 struct xsave_struct
{
165 uint8_t fpu_sse
[512];
166 struct xsave_hdr_struct xsave_hdr
;
169 struct bndregs_struct bndregs
;
170 struct bndcsr_struct bndcsr
;
171 } __attribute__((packed
));
173 uint8_t __attribute__((__aligned__(64))) buffer
[4096];
174 struct xsave_struct
*xsave_buf
= (struct xsave_struct
*)buffer
;
176 uint8_t __attribute__((__aligned__(64))) test_buffer
[4096];
177 struct xsave_struct
*xsave_test_buf
= (struct xsave_struct
*)test_buffer
;
179 uint64_t num_bnd_chk
;
181 static __always_inline
void xrstor_state(struct xsave_struct
*fx
, uint64_t mask
)
183 uint32_t lmask
= mask
;
184 uint32_t hmask
= mask
>> 32;
186 asm volatile(".byte " REX_PREFIX
"0x0f,0xae,0x2f\n\t"
187 : : "D" (fx
), "m" (*fx
), "a" (lmask
), "d" (hmask
)
191 static __always_inline
void xsave_state_1(void *_fx
, uint64_t mask
)
193 uint32_t lmask
= mask
;
194 uint32_t hmask
= mask
>> 32;
195 unsigned char *fx
= _fx
;
197 asm volatile(".byte " REX_PREFIX
"0x0f,0xae,0x27\n\t"
198 : : "D" (fx
), "m" (*fx
), "a" (lmask
), "d" (hmask
)
202 static inline uint64_t xgetbv(uint32_t index
)
206 asm volatile(".byte 0x0f,0x01,0xd0" /* xgetbv */
207 : "=a" (eax
), "=d" (edx
)
209 return eax
+ ((uint64_t)edx
<< 32);
212 static uint64_t read_mpx_status_sig(ucontext_t
*uctxt
)
214 memset(buffer
, 0, sizeof(buffer
));
216 (uint8_t *)uctxt
->uc_mcontext
.fpregs
+ XSAVE_OFFSET_IN_FPMEM
,
217 sizeof(struct xsave_struct
));
219 return xsave_buf
->bndcsr
.status_reg
;
224 static uint8_t *get_next_inst_ip(uint8_t *addr
)
233 /* determine the prefix. */
242 /* look for rex prefix */
243 if ((*ip
& 0x40) == 0x40)
246 /* Make sure we have a MPX instruction. */
250 /* Skip the op code byte. */
253 /* Get the modrm byte. */
256 /* Break it down into parts. */
260 /* Init the parts of the address mode. */
263 /* Is it a mem mode? */
265 /* look for scaled indexed addressing */
286 /* MODRM addressing */
289 /* DISP32 addressing, no base */
308 static inline void *__si_bounds_lower(siginfo_t
*si
)
313 static inline void *__si_bounds_upper(siginfo_t
*si
)
320 * This deals with old version of _sigfault in some distros:
344 static inline void **__si_bounds_hack(siginfo_t
*si
)
346 void *sigfault
= &si
->_sifields
._sigfault
;
347 void *end_sigfault
= sigfault
+ sizeof(si
->_sifields
._sigfault
);
348 int *trapno
= (int*)end_sigfault
;
349 /* skip _trapno and _addr_lsb */
350 void **__si_lower
= (void**)(trapno
+ 2);
355 static inline void *__si_bounds_lower(siginfo_t
*si
)
357 return *__si_bounds_hack(si
);
360 static inline void *__si_bounds_upper(siginfo_t
*si
)
362 return *(__si_bounds_hack(si
) + 1);
367 static int expected_bnd_index
= -1;
368 uint64_t shadow_plb
[NR_MPX_BOUNDS_REGISTERS
][2]; /* shadow MPX bound registers */
369 unsigned long shadow_map
[NR_MPX_BOUNDS_REGISTERS
];
371 /* Failed address bound checks: */
373 # define SEGV_BNDERR 3
377 * The kernel is supposed to provide some information about the bounds
378 * exception in the siginfo. It should match what we have in the bounds
379 * registers that we are checking against. Just check against the shadow copy
380 * since it is easily available, and we also check that *it* matches the real
383 void check_siginfo_vs_shadow(siginfo_t
* si
)
386 void *shadow_lower
= (void *)(unsigned long)shadow_plb
[expected_bnd_index
][0];
387 void *shadow_upper
= (void *)(unsigned long)shadow_plb
[expected_bnd_index
][1];
389 if ((expected_bnd_index
< 0) ||
390 (expected_bnd_index
>= NR_MPX_BOUNDS_REGISTERS
)) {
391 fprintf(stderr
, "ERROR: invalid expected_bnd_index: %d\n",
395 if (__si_bounds_lower(si
) != shadow_lower
)
397 if (__si_bounds_upper(si
) != shadow_upper
)
401 fprintf(stderr
, "ERROR: siginfo bounds do not match "
402 "shadow bounds for register %d\n", expected_bnd_index
);
407 void handler(int signum
, siginfo_t
*si
, void *vucontext
)
410 ucontext_t
*uctxt
= vucontext
;
414 dprintf1("entered signal handler\n");
416 trapno
= uctxt
->uc_mcontext
.gregs
[REG_TRAPNO
];
417 ip
= uctxt
->uc_mcontext
.gregs
[REG_IP_IDX
];
420 typeof(si
->si_addr
) *si_addr_ptr
= &si
->si_addr
;
421 uint64_t status
= read_mpx_status_sig(uctxt
);
422 uint64_t br_reason
= status
& 0x3;
425 dprintf1("#BR 0x%jx (total seen: %d)\n", status
, br_count
);
427 dprintf2("Saw a #BR! status 0x%jx at %016lx br_reason: %jx\n",
428 status
, ip
, br_reason
);
429 dprintf2("si_signo: %d\n", si
->si_signo
);
430 dprintf2(" signum: %d\n", signum
);
431 dprintf2("info->si_code == SEGV_BNDERR: %d\n",
432 (si
->si_code
== SEGV_BNDERR
));
433 dprintf2("info->si_code: %d\n", si
->si_code
);
434 dprintf2("info->si_lower: %p\n", __si_bounds_lower(si
));
435 dprintf2("info->si_upper: %p\n", __si_bounds_upper(si
));
437 for (i
= 0; i
< 8; i
++)
438 dprintf3("[%d]: %p\n", i
, si_addr_ptr
[i
]);
440 case 0: /* traditional BR */
442 "Undefined status with bound exception:%jx\n",
445 case 1: /* #BR MPX bounds exception */
446 /* these are normal and we expect to see them */
448 check_siginfo_vs_shadow(si
);
450 dprintf1("bounds exception (normal): status 0x%jx at %p si_addr: %p\n",
451 status
, (void *)ip
, si
->si_addr
);
453 uctxt
->uc_mcontext
.gregs
[REG_IP_IDX
] =
454 (greg_t
)get_next_inst_ip((uint8_t *)ip
);
457 fprintf(stderr
, "#BR status == 2, missing bounds table,"
458 "kernel should have handled!!\n");
462 fprintf(stderr
, "bound check error: status 0x%jx at %p\n",
465 uctxt
->uc_mcontext
.gregs
[REG_IP_IDX
] =
466 (greg_t
)get_next_inst_ip((uint8_t *)ip
);
467 fprintf(stderr
, "bound check error: si_addr %p\n", si
->si_addr
);
470 } else if (trapno
== 14) {
471 eprintf("ERROR: In signal handler, page fault, trapno = %d, ip = %016lx\n",
473 eprintf("si_addr %p\n", si
->si_addr
);
474 eprintf("REG_ERR: %lx\n", (unsigned long)uctxt
->uc_mcontext
.gregs
[REG_ERR
]);
477 eprintf("unexpected trap %d! at 0x%lx\n", trapno
, ip
);
478 eprintf("si_addr %p\n", si
->si_addr
);
479 eprintf("REG_ERR: %lx\n", (unsigned long)uctxt
->uc_mcontext
.gregs
[REG_ERR
]);
484 static inline void cpuid_count(unsigned int op
, int count
,
485 unsigned int *eax
, unsigned int *ebx
,
486 unsigned int *ecx
, unsigned int *edx
)
490 __cpuid(eax
, ebx
, ecx
, edx
);
493 #define XSTATE_CPUID 0x0000000d
496 * List of XSAVE features Linux knows about:
505 XSTATE_BIT_ZMM_Hi256
,
511 #define XSTATE_FP (1 << XSTATE_BIT_FP)
512 #define XSTATE_SSE (1 << XSTATE_BIT_SSE)
513 #define XSTATE_YMM (1 << XSTATE_BIT_YMM)
514 #define XSTATE_BNDREGS (1 << XSTATE_BIT_BNDREGS)
515 #define XSTATE_BNDCSR (1 << XSTATE_BIT_BNDCSR)
516 #define XSTATE_OPMASK (1 << XSTATE_BIT_OPMASK)
517 #define XSTATE_ZMM_Hi256 (1 << XSTATE_BIT_ZMM_Hi256)
518 #define XSTATE_Hi16_ZMM (1 << XSTATE_BIT_Hi16_ZMM)
520 #define MPX_XSTATES (XSTATE_BNDREGS | XSTATE_BNDCSR) /* 0x18 */
522 bool one_bit(unsigned int x
, int bit
)
524 return !!(x
& (1<<bit
));
527 void print_state_component(int state_bit_nr
, char *name
)
529 unsigned int eax
, ebx
, ecx
, edx
;
530 unsigned int state_component_size
;
531 unsigned int state_component_supervisor
;
532 unsigned int state_component_user
;
533 unsigned int state_component_aligned
;
535 /* See SDM Section 13.2 */
536 cpuid_count(XSTATE_CPUID
, state_bit_nr
, &eax
, &ebx
, &ecx
, &edx
);
537 assert(eax
|| ebx
|| ecx
);
538 state_component_size
= eax
;
539 state_component_supervisor
= ((!ebx
) && one_bit(ecx
, 0));
540 state_component_user
= !one_bit(ecx
, 0);
541 state_component_aligned
= one_bit(ecx
, 1);
542 printf("%8s: size: %d user: %d supervisor: %d aligned: %d\n",
544 state_component_size
, state_component_user
,
545 state_component_supervisor
, state_component_aligned
);
549 /* Intel-defined CPU features, CPUID level 0x00000001 (ecx) */
550 #define XSAVE_FEATURE_BIT (26) /* XSAVE/XRSTOR/XSETBV/XGETBV */
551 #define OSXSAVE_FEATURE_BIT (27) /* XSAVE enabled in the OS */
553 bool check_mpx_support(void)
555 unsigned int eax
, ebx
, ecx
, edx
;
557 cpuid_count(1, 0, &eax
, &ebx
, &ecx
, &edx
);
559 /* We can't do much without XSAVE, so just make these assert()'s */
560 if (!one_bit(ecx
, XSAVE_FEATURE_BIT
)) {
561 fprintf(stderr
, "processor lacks XSAVE, can not run MPX tests\n");
565 if (!one_bit(ecx
, OSXSAVE_FEATURE_BIT
)) {
566 fprintf(stderr
, "processor lacks OSXSAVE, can not run MPX tests\n");
570 /* CPUs not supporting the XSTATE CPUID leaf do not support MPX */
571 /* Is this redundant with the feature bit checks? */
572 cpuid_count(0, 0, &eax
, &ebx
, &ecx
, &edx
);
573 if (eax
< XSTATE_CPUID
) {
574 fprintf(stderr
, "processor lacks XSTATE CPUID leaf,"
575 " can not run MPX tests\n");
579 printf("XSAVE is supported by HW & OS\n");
581 cpuid_count(XSTATE_CPUID
, 0, &eax
, &ebx
, &ecx
, &edx
);
583 printf("XSAVE processor supported state mask: 0x%x\n", eax
);
584 printf("XSAVE OS supported state mask: 0x%jx\n", xgetbv(0));
586 /* Make sure that the MPX states are enabled in in XCR0 */
587 if ((eax
& MPX_XSTATES
) != MPX_XSTATES
) {
588 fprintf(stderr
, "processor lacks MPX XSTATE(s), can not run MPX tests\n");
592 /* Make sure the MPX states are supported by XSAVE* */
593 if ((xgetbv(0) & MPX_XSTATES
) != MPX_XSTATES
) {
594 fprintf(stderr
, "MPX XSTATE(s) no enabled in XCR0, "
595 "can not run MPX tests\n");
599 print_state_component(XSTATE_BIT_BNDREGS
, "BNDREGS");
600 print_state_component(XSTATE_BIT_BNDCSR
, "BNDCSR");
605 void enable_mpx(void *l1base
)
607 /* enable point lookup */
608 memset(buffer
, 0, sizeof(buffer
));
609 xrstor_state(xsave_buf
, 0x18);
611 xsave_buf
->xsave_hdr
.xstate_bv
= 0x10;
612 xsave_buf
->bndcsr
.cfg_reg_u
= (unsigned long)l1base
| 1;
613 xsave_buf
->bndcsr
.status_reg
= 0;
615 dprintf2("bf xrstor\n");
616 dprintf2("xsave cndcsr: status %jx, configu %jx\n",
617 xsave_buf
->bndcsr
.status_reg
, xsave_buf
->bndcsr
.cfg_reg_u
);
618 xrstor_state(xsave_buf
, 0x18);
619 dprintf2("after xrstor\n");
621 xsave_state_1(xsave_buf
, 0x18);
623 dprintf1("xsave bndcsr: status %jx, configu %jx\n",
624 xsave_buf
->bndcsr
.status_reg
, xsave_buf
->bndcsr
.cfg_reg_u
);
627 #include <sys/prctl.h>
629 struct mpx_bounds_dir
*bounds_dir_ptr
;
631 unsigned long __bd_incore(const char *func
, int line
)
633 unsigned long ret
= nr_incore(bounds_dir_ptr
, MPX_BOUNDS_DIR_SIZE_BYTES
);
636 #define bd_incore() __bd_incore(__func__, __LINE__)
638 void check_clear(void *ptr
, unsigned long sz
)
642 for (i
= ptr
; (void *)i
< ptr
+ sz
; i
++) {
644 dprintf1("%p is NOT clear at %p\n", ptr
, i
);
648 dprintf1("%p is clear for %lx\n", ptr
, sz
);
651 void check_clear_bd(void)
653 check_clear(bounds_dir_ptr
, 2UL << 30);
656 #define USE_MALLOC_FOR_BOUNDS_DIR 1
657 bool process_specific_init(void)
661 /* Guarantee we have the space to align it, add padding: */
662 unsigned long pad
= getpagesize();
664 size
= 2UL << 30; /* 2GB */
665 if (sizeof(unsigned long) == 4)
666 size
= 4UL << 20; /* 4MB */
667 dprintf1("trying to allocate %ld MB bounds directory\n", (size
>> 20));
669 if (USE_MALLOC_FOR_BOUNDS_DIR
) {
672 dir
= malloc(size
+ pad
);
674 _dir
= (unsigned long)dir
;
680 * This makes debugging easier because the address
681 * calculations are simpler:
683 dir
= mmap((void *)0x200000000000, size
+ pad
,
684 PROT_READ
|PROT_WRITE
,
685 MAP_ANONYMOUS
|MAP_PRIVATE
, -1, 0);
686 if (dir
== (void *)-1) {
687 perror("unable to allocate bounds directory");
690 check_clear(dir
, size
);
692 bounds_dir_ptr
= (void *)dir
;
693 madvise(bounds_dir_ptr
, size
, MADV_NOHUGEPAGE
);
695 dprintf1("bounds directory: 0x%p -> 0x%p\n", bounds_dir_ptr
,
696 (char *)bounds_dir_ptr
+ size
);
697 check_clear(dir
, size
);
699 check_clear(dir
, size
);
700 if (prctl(43, 0, 0, 0, 0)) {
701 printf("no MPX support\n");
708 bool process_specific_finish(void)
711 printf("no MPX support\n");
720 struct sigaction newact
;
721 struct sigaction oldact
;
723 /* #BR is mapped to sigsegv */
724 int signum
= SIGSEGV
;
726 newact
.sa_handler
= 0; /* void(*)(int)*/
727 newact
.sa_sigaction
= handler
; /* void (*)(int, siginfo_t*, void *) */
729 /*sigset_t - signals to block while in the handler */
730 /* get the old signal mask. */
731 rs
= sigprocmask(SIG_SETMASK
, 0, &newact
.sa_mask
);
734 /* call sa_sigaction, not sa_handler*/
735 newact
.sa_flags
= SA_SIGINFO
;
737 newact
.sa_restorer
= 0; /* void(*)(), obsolete */
738 r
= sigaction(signum
, &newact
, &oldact
);
742 void mpx_prepare(void)
744 dprintf2("%s()\n", __func__
);
746 process_specific_init();
749 void mpx_cleanup(void)
751 printf("%s(): %jd BRs. bye...\n", __func__
, num_bnd_chk
);
752 process_specific_finish();
755 /*-------------- the following is test case ---------------*/
762 uint64_t num_lower_brs
;
763 uint64_t num_upper_brs
;
765 #define MPX_CONFIG_OFFSET 1024
766 #define MPX_BOUNDS_OFFSET 960
767 #define MPX_HEADER_OFFSET 512
768 #define MAX_ADDR_TESTED (1<<28)
769 #define TEST_ROUNDS 100
773 0F 1B /r BNDSTX-Store Extended Bounds Using Address Translation
774 66 0F 1A /r BNDMOV bnd1, bnd2/m128
775 66 0F 1B /r BNDMOV bnd1/m128, bnd2
776 F2 0F 1A /r BNDCU bnd, r/m64
777 F2 0F 1B /r BNDCN bnd, r/m64
778 F3 0F 1A /r BNDCL bnd, r/m64
779 F3 0F 1B /r BNDMK bnd, m64
782 static __always_inline
void xsave_state(void *_fx
, uint64_t mask
)
784 uint32_t lmask
= mask
;
785 uint32_t hmask
= mask
>> 32;
786 unsigned char *fx
= _fx
;
788 asm volatile(".byte " REX_PREFIX
"0x0f,0xae,0x27\n\t"
789 : : "D" (fx
), "m" (*fx
), "a" (lmask
), "d" (hmask
)
793 static __always_inline
void mpx_clear_bnd0(void)
797 /* F3 0F 1B /r BNDMK bnd, m64 */
798 /* f3 0f 1b 04 11 bndmk (%rcx,%rdx,1),%bnd0 */
799 asm volatile(".byte 0xf3,0x0f,0x1b,0x04,0x11\n\t"
800 : : "c" (ptr
), "d" (size
-1)
804 static __always_inline
void mpx_make_bound_helper(unsigned long ptr
,
807 /* F3 0F 1B /r BNDMK bnd, m64 */
808 /* f3 0f 1b 04 11 bndmk (%rcx,%rdx,1),%bnd0 */
809 asm volatile(".byte 0xf3,0x0f,0x1b,0x04,0x11\n\t"
810 : : "c" (ptr
), "d" (size
-1)
814 static __always_inline
void mpx_check_lowerbound_helper(unsigned long ptr
)
816 /* F3 0F 1A /r NDCL bnd, r/m64 */
817 /* f3 0f 1a 01 bndcl (%rcx),%bnd0 */
818 asm volatile(".byte 0xf3,0x0f,0x1a,0x01\n\t"
823 static __always_inline
void mpx_check_upperbound_helper(unsigned long ptr
)
825 /* F2 0F 1A /r BNDCU bnd, r/m64 */
826 /* f2 0f 1a 01 bndcu (%rcx),%bnd0 */
827 asm volatile(".byte 0xf2,0x0f,0x1a,0x01\n\t"
832 static __always_inline
void mpx_movbndreg_helper()
834 /* 66 0F 1B /r BNDMOV bnd1/m128, bnd2 */
835 /* 66 0f 1b c2 bndmov %bnd0,%bnd2 */
837 asm volatile(".byte 0x66,0x0f,0x1b,0xc2\n\t");
840 static __always_inline
void mpx_movbnd2mem_helper(uint8_t *mem
)
842 /* 66 0F 1B /r BNDMOV bnd1/m128, bnd2 */
843 /* 66 0f 1b 01 bndmov %bnd0,(%rcx) */
844 asm volatile(".byte 0x66,0x0f,0x1b,0x01\n\t"
849 static __always_inline
void mpx_movbnd_from_mem_helper(uint8_t *mem
)
851 /* 66 0F 1A /r BNDMOV bnd1, bnd2/m128 */
852 /* 66 0f 1a 01 bndmov (%rcx),%bnd0 */
853 asm volatile(".byte 0x66,0x0f,0x1a,0x01\n\t"
858 static __always_inline
void mpx_store_dsc_helper(unsigned long ptr_addr
,
859 unsigned long ptr_val
)
861 /* 0F 1B /r BNDSTX-Store Extended Bounds Using Address Translation */
862 /* 0f 1b 04 11 bndstx %bnd0,(%rcx,%rdx,1) */
863 asm volatile(".byte 0x0f,0x1b,0x04,0x11\n\t"
864 : : "c" (ptr_addr
), "d" (ptr_val
)
868 static __always_inline
void mpx_load_dsc_helper(unsigned long ptr_addr
,
869 unsigned long ptr_val
)
871 /* 0F 1A /r BNDLDX-Load */
872 /*/ 0f 1a 04 11 bndldx (%rcx,%rdx,1),%bnd0 */
873 asm volatile(".byte 0x0f,0x1a,0x04,0x11\n\t"
874 : : "c" (ptr_addr
), "d" (ptr_val
)
878 void __print_context(void *__print_xsave_buffer
, int line
)
880 uint64_t *bounds
= (uint64_t *)(__print_xsave_buffer
+ MPX_BOUNDS_OFFSET
);
881 uint64_t *cfg
= (uint64_t *)(__print_xsave_buffer
+ MPX_CONFIG_OFFSET
);
884 eprintf("%s()::%d\n", "print_context", line
);
885 for (i
= 0; i
< 4; i
++) {
886 eprintf("bound[%d]: 0x%016lx 0x%016lx(0x%016lx)\n", i
,
887 (unsigned long)bounds
[i
*2],
888 ~(unsigned long)bounds
[i
*2+1],
889 (unsigned long)bounds
[i
*2+1]);
892 eprintf("cpcfg: %jx cpstatus: %jx\n", cfg
[0], cfg
[1]);
894 #define print_context(x) __print_context(x, __LINE__)
896 #define dprint_context(x) print_context(x)
898 #define dprint_context(x) do{}while(0)
905 srand((unsigned int)time(NULL
));
907 for (i
= 0; i
< 4; i
++) {
908 shadow_plb
[i
][0] = 0;
909 shadow_plb
[i
][1] = ~(unsigned long)0;
913 long int __mpx_random(int line
)
916 static long fake
= 722122311;
923 #define mpx_random() __mpx_random(__LINE__)
925 uint8_t *get_random_addr()
927 uint8_t*addr
= (uint8_t *)(unsigned long)(rand() % MAX_ADDR_TESTED
);
928 return (addr
- (unsigned long)addr
% sizeof(uint8_t *));
931 static inline bool compare_context(void *__xsave_buffer
)
933 uint64_t *bounds
= (uint64_t *)(__xsave_buffer
+ MPX_BOUNDS_OFFSET
);
936 for (i
= 0; i
< 4; i
++) {
937 dprintf3("shadow[%d]{%016lx/%016lx}\nbounds[%d]{%016lx/%016lx}\n",
938 i
, (unsigned long)shadow_plb
[i
][0], (unsigned long)shadow_plb
[i
][1],
939 i
, (unsigned long)bounds
[i
*2], ~(unsigned long)bounds
[i
*2+1]);
940 if ((shadow_plb
[i
][0] != bounds
[i
*2]) ||
941 (shadow_plb
[i
][1] != ~(unsigned long)bounds
[i
*2+1])) {
942 eprintf("ERROR comparing shadow to real bound register %d\n", i
);
943 eprintf("shadow{0x%016lx/0x%016lx}\nbounds{0x%016lx/0x%016lx}\n",
944 (unsigned long)shadow_plb
[i
][0], (unsigned long)shadow_plb
[i
][1],
945 (unsigned long)bounds
[i
*2], (unsigned long)bounds
[i
*2+1]);
953 void mkbnd_shadow(uint8_t *ptr
, int index
, long offset
)
955 uint64_t *lower
= (uint64_t *)&(shadow_plb
[index
][0]);
956 uint64_t *upper
= (uint64_t *)&(shadow_plb
[index
][1]);
957 *lower
= (unsigned long)ptr
;
958 *upper
= (unsigned long)ptr
+ offset
- 1;
961 void check_lowerbound_shadow(uint8_t *ptr
, int index
)
963 uint64_t *lower
= (uint64_t *)&(shadow_plb
[index
][0]);
964 if (*lower
> (uint64_t)(unsigned long)ptr
)
967 dprintf1("LowerBoundChk passed:%p\n", ptr
);
970 void check_upperbound_shadow(uint8_t *ptr
, int index
)
972 uint64_t upper
= *(uint64_t *)&(shadow_plb
[index
][1]);
973 if (upper
< (uint64_t)(unsigned long)ptr
)
976 dprintf1("UpperBoundChk passed:%p\n", ptr
);
979 __always_inline
void movbndreg_shadow(int src
, int dest
)
981 shadow_plb
[dest
][0] = shadow_plb
[src
][0];
982 shadow_plb
[dest
][1] = shadow_plb
[src
][1];
985 __always_inline
void movbnd2mem_shadow(int src
, unsigned long *dest
)
987 unsigned long *lower
= (unsigned long *)&(shadow_plb
[src
][0]);
988 unsigned long *upper
= (unsigned long *)&(shadow_plb
[src
][1]);
993 __always_inline
void movbnd_from_mem_shadow(unsigned long *src
, int dest
)
995 unsigned long *lower
= (unsigned long *)&(shadow_plb
[dest
][0]);
996 unsigned long *upper
= (unsigned long *)&(shadow_plb
[dest
][1]);
1001 __always_inline
void stdsc_shadow(int index
, uint8_t *ptr
, uint8_t *ptr_val
)
1003 shadow_map
[0] = (unsigned long)shadow_plb
[index
][0];
1004 shadow_map
[1] = (unsigned long)shadow_plb
[index
][1];
1005 shadow_map
[2] = (unsigned long)ptr_val
;
1006 dprintf3("%s(%d, %p, %p) set shadow map[2]: %p\n", __func__
,
1007 index
, ptr
, ptr_val
, ptr_val
);
1011 void lddsc_shadow(int index
, uint8_t *ptr
, uint8_t *ptr_val
)
1013 uint64_t lower
= shadow_map
[0];
1014 uint64_t upper
= shadow_map
[1];
1015 uint8_t *value
= (uint8_t *)shadow_map
[2];
1017 if (value
!= ptr_val
) {
1018 dprintf2("%s(%d, %p, %p) init shadow bounds[%d] "
1019 "because %p != %p\n", __func__
, index
, ptr
,
1020 ptr_val
, index
, value
, ptr_val
);
1021 shadow_plb
[index
][0] = 0;
1022 shadow_plb
[index
][1] = ~(unsigned long)0;
1024 shadow_plb
[index
][0] = lower
;
1025 shadow_plb
[index
][1] = upper
;
1030 static __always_inline
void mpx_test_helper0(uint8_t *buf
, uint8_t *ptr
)
1032 mpx_make_bound_helper((unsigned long)ptr
, 0x1800);
1035 static __always_inline
void mpx_test_helper0_shadow(uint8_t *buf
, uint8_t *ptr
)
1037 mkbnd_shadow(ptr
, 0, 0x1800);
1040 static __always_inline
void mpx_test_helper1(uint8_t *buf
, uint8_t *ptr
)
1042 /* these are hard-coded to check bnd0 */
1043 expected_bnd_index
= 0;
1044 mpx_check_lowerbound_helper((unsigned long)(ptr
-1));
1045 mpx_check_upperbound_helper((unsigned long)(ptr
+0x1800));
1046 /* reset this since we do not expect any more bounds exceptions */
1047 expected_bnd_index
= -1;
1050 static __always_inline
void mpx_test_helper1_shadow(uint8_t *buf
, uint8_t *ptr
)
1052 check_lowerbound_shadow(ptr
-1, 0);
1053 check_upperbound_shadow(ptr
+0x1800, 0);
1056 static __always_inline
void mpx_test_helper2(uint8_t *buf
, uint8_t *ptr
)
1058 mpx_make_bound_helper((unsigned long)ptr
, 0x1800);
1059 mpx_movbndreg_helper();
1060 mpx_movbnd2mem_helper(buf
);
1061 mpx_make_bound_helper((unsigned long)(ptr
+0x12), 0x1800);
1064 static __always_inline
void mpx_test_helper2_shadow(uint8_t *buf
, uint8_t *ptr
)
1066 mkbnd_shadow(ptr
, 0, 0x1800);
1067 movbndreg_shadow(0, 2);
1068 movbnd2mem_shadow(0, (unsigned long *)buf
);
1069 mkbnd_shadow(ptr
+0x12, 0, 0x1800);
1072 static __always_inline
void mpx_test_helper3(uint8_t *buf
, uint8_t *ptr
)
1074 mpx_movbnd_from_mem_helper(buf
);
1077 static __always_inline
void mpx_test_helper3_shadow(uint8_t *buf
, uint8_t *ptr
)
1079 movbnd_from_mem_shadow((unsigned long *)buf
, 0);
1082 static __always_inline
void mpx_test_helper4(uint8_t *buf
, uint8_t *ptr
)
1084 mpx_store_dsc_helper((unsigned long)buf
, (unsigned long)ptr
);
1085 mpx_make_bound_helper((unsigned long)(ptr
+0x12), 0x1800);
1088 static __always_inline
void mpx_test_helper4_shadow(uint8_t *buf
, uint8_t *ptr
)
1090 stdsc_shadow(0, buf
, ptr
);
1091 mkbnd_shadow(ptr
+0x12, 0, 0x1800);
1094 static __always_inline
void mpx_test_helper5(uint8_t *buf
, uint8_t *ptr
)
1096 mpx_load_dsc_helper((unsigned long)buf
, (unsigned long)ptr
);
1099 static __always_inline
void mpx_test_helper5_shadow(uint8_t *buf
, uint8_t *ptr
)
1101 lddsc_shadow(0, buf
, ptr
);
1104 #define NR_MPX_TEST_FUNCTIONS 6
1107 * For compatibility reasons, MPX will clear the bounds registers
1108 * when you make function calls (among other things). We have to
1109 * preserve the registers in between calls to the "helpers" since
1110 * they build on each other.
1112 * Be very careful not to make any function calls inside the
1113 * helpers, or anywhere else beween the xrstor and xsave.
1115 #define run_helper(helper_nr, buf, buf_shadow, ptr) do { \
1116 xrstor_state(xsave_test_buf, flags); \
1117 mpx_test_helper##helper_nr(buf, ptr); \
1118 xsave_state(xsave_test_buf, flags); \
1119 mpx_test_helper##helper_nr##_shadow(buf_shadow, ptr); \
1122 static void run_helpers(int nr
, uint8_t *buf
, uint8_t *buf_shadow
, uint8_t *ptr
)
1124 uint64_t flags
= 0x18;
1126 dprint_context(xsave_test_buf
);
1129 run_helper(0, buf
, buf_shadow
, ptr
);
1132 run_helper(1, buf
, buf_shadow
, ptr
);
1135 run_helper(2, buf
, buf_shadow
, ptr
);
1138 run_helper(3, buf
, buf_shadow
, ptr
);
1141 run_helper(4, buf
, buf_shadow
, ptr
);
1144 run_helper(5, buf
, buf_shadow
, ptr
);
1150 dprint_context(xsave_test_buf
);
1153 unsigned long buf_shadow
[1024]; /* used to check load / store descriptors */
1154 extern long inspect_me(struct mpx_bounds_dir
*bounds_dir
);
1156 long cover_buf_with_bt_entries(void *buf
, long buf_len
)
1161 unsigned long buf_len_in_ptrs
;
1163 /* Fill about 1/100 of the space with bt entries */
1164 nr_to_fill
= buf_len
/ (sizeof(unsigned long) * ratio
);
1167 dprintf3("%s() nr_to_fill: %ld\n", __func__
, nr_to_fill
);
1169 /* Align the buffer to pointer size */
1170 while (((unsigned long)buf
) % sizeof(void *)) {
1174 /* We are storing pointers, so make */
1175 buf_len_in_ptrs
= buf_len
/ sizeof(void *);
1177 for (i
= 0; i
< nr_to_fill
; i
++) {
1178 long index
= (mpx_random() % buf_len_in_ptrs
);
1179 void *ptr
= buf
+ index
* sizeof(unsigned long);
1180 unsigned long ptr_addr
= (unsigned long)ptr
;
1182 /* ptr and size can be anything */
1183 mpx_make_bound_helper((unsigned long)ptr
, 8);
1186 * take bnd0 and put it in to bounds tables "buf + index" is an
1187 * address inside the buffer where we are pretending that we
1188 * are going to put a pointer We do not, though because we will
1189 * never load entries from the table, so it doesn't matter.
1191 mpx_store_dsc_helper(ptr_addr
, (unsigned long)ptr
);
1192 dprintf4("storing bound table entry for %lx (buf start @ %p)\n",
1198 unsigned long align_down(unsigned long alignme
, unsigned long align_to
)
1200 return alignme
& ~(align_to
-1);
1203 unsigned long align_up(unsigned long alignme
, unsigned long align_to
)
1205 return (alignme
+ align_to
- 1) & ~(align_to
-1);
1209 * Using 1MB alignment guarantees that each no allocation
1210 * will overlap with another's bounds tables.
1212 * We have to cook our own allocator here. malloc() can
1213 * mix other allocation with ours which means that even
1214 * if we free all of our allocations, there might still
1215 * be bounds tables for the *areas* since there is other
1216 * valid memory there.
1218 * We also can't use malloc() because a free() of an area
1219 * might not free it back to the kernel. We want it
1220 * completely unmapped an malloc() does not guarantee
1224 long alignment
= 4096;
1225 long sz_alignment
= 4096;
1227 long alignment
= 1 * MB
;
1228 long sz_alignment
= 1 * MB
;
1230 void *mpx_mini_alloc(unsigned long sz
)
1232 unsigned long long tries
= 0;
1237 sz
= align_up(sz
, sz_alignment
);
1239 try_at
= last
+ alignment
;
1241 ptr
= mmap(try_at
, sz
, PROT_READ
|PROT_WRITE
,
1242 MAP_ANONYMOUS
|MAP_PRIVATE
, -1, 0);
1243 if (ptr
== (void *)-1)
1249 try_at
+= alignment
;
1252 * This isn't quite correct for 32-bit binaries
1253 * on 64-bit kernels since they can use the
1254 * entire 32-bit address space, but it's close
1257 if (try_at
> (void *)0xC0000000)
1259 if (try_at
> (void *)0x0000800000000000)
1261 try_at
= (void *)0x0;
1262 if (!(++tries
% 10000))
1263 dprintf1("stuck in %s(), tries: %lld\n", __func__
, tries
);
1267 dprintf3("mpx_mini_alloc(0x%lx) returning: %p\n", sz
, ptr
);
1270 void mpx_mini_free(void *ptr
, long sz
)
1272 dprintf2("%s() ptr: %p\n", __func__
, ptr
);
1273 if ((unsigned long)ptr
> 0x100000000000) {
1274 dprintf1("uh oh !!!!!!!!!!!!!!! pointer too high: %p\n", ptr
);
1277 sz
= align_up(sz
, sz_alignment
);
1278 dprintf3("%s() ptr: %p before munmap\n", __func__
, ptr
);
1280 dprintf3("%s() ptr: %p DONE\n", __func__
, ptr
);
1283 #define NR_MALLOCS 100
1289 struct one_malloc mallocs
[NR_MALLOCS
];
1291 void free_one_malloc(int index
)
1293 unsigned long free_ptr
;
1296 if (!mallocs
[index
].ptr
)
1299 mpx_mini_free(mallocs
[index
].ptr
, mallocs
[index
].size
);
1300 dprintf4("freed[%d]: %p\n", index
, mallocs
[index
].ptr
);
1302 free_ptr
= (unsigned long)mallocs
[index
].ptr
;
1304 dprintf4("lowerbits: %lx / %lx mask: %lx\n", free_ptr
,
1305 (free_ptr
& mask
), mask
);
1306 assert((free_ptr
& mask
) == 0);
1308 mallocs
[index
].ptr
= NULL
;
1312 #define MPX_BOUNDS_TABLE_COVERS 4096
1314 #define MPX_BOUNDS_TABLE_COVERS (1 * MB)
1316 void zap_everything(void)
1322 before_zap
= inspect_me(bounds_dir_ptr
);
1323 dprintf1("zapping everything start: %ld\n", before_zap
);
1324 for (i
= 0; i
< NR_MALLOCS
; i
++)
1327 after_zap
= inspect_me(bounds_dir_ptr
);
1328 dprintf1("zapping everything done: %ld\n", after_zap
);
1330 * We only guarantee to empty the thing out if our allocations are
1331 * exactly aligned on the boundaries of a boudns table.
1333 if ((alignment
>= MPX_BOUNDS_TABLE_COVERS
) &&
1334 (sz_alignment
>= MPX_BOUNDS_TABLE_COVERS
)) {
1338 assert(after_zap
== 0);
1342 void do_one_malloc(void)
1344 static int malloc_counter
;
1346 int rand_index
= (mpx_random() % NR_MALLOCS
);
1347 void *ptr
= mallocs
[rand_index
].ptr
;
1349 dprintf3("%s() enter\n", __func__
);
1352 dprintf3("freeing one malloc at index: %d\n", rand_index
);
1353 free_one_malloc(rand_index
);
1354 if (mpx_random() % (NR_MALLOCS
*3) == 3) {
1356 dprintf3("zapping some more\n");
1357 for (i
= rand_index
; i
< NR_MALLOCS
; i
++)
1360 if ((mpx_random() % zap_all_every_this_many_mallocs
) == 4)
1365 sz
= (1 + mpx_random() % 1000) * 1000;
1366 ptr
= mpx_mini_alloc(sz
);
1369 * If we are failing allocations, just assume we
1370 * are out of memory and zap everything.
1372 dprintf3("zapping everything because out of memory\n");
1377 dprintf3("malloc: %p size: 0x%lx\n", ptr
, sz
);
1378 mallocs
[rand_index
].nr_filled_btes
= cover_buf_with_bt_entries(ptr
, sz
);
1379 mallocs
[rand_index
].ptr
= ptr
;
1380 mallocs
[rand_index
].size
= sz
;
1382 if ((++malloc_counter
) % inspect_every_this_many_mallocs
== 0)
1383 inspect_me(bounds_dir_ptr
);
1386 void run_timed_test(void (*test_func
)(void))
1390 static time_t last_print
;
1397 if ((now
- start
) > TEST_DURATION_SECS
)
1403 if ((now
- last_print
> 1) || done
) {
1404 printf("iteration %ld complete, OK so far\n", iteration
);
1410 void check_bounds_table_frees(void)
1412 printf("executing unmaptest\n");
1413 inspect_me(bounds_dir_ptr
);
1414 run_timed_test(&do_one_malloc
);
1415 printf("done with malloc() fun\n");
1418 void insn_test_failed(int test_nr
, int test_round
, void *buf
,
1419 void *buf_shadow
, void *ptr
)
1421 print_context(xsave_test_buf
);
1422 eprintf("ERROR: test %d round %d failed\n", test_nr
, test_round
);
1423 while (test_nr
== 5) {
1424 struct mpx_bt_entry
*bte
;
1425 struct mpx_bounds_dir
*bd
= (void *)bounds_dir_ptr
;
1426 struct mpx_bd_entry
*bde
= mpx_vaddr_to_bd_entry(buf
, bd
);
1428 printf(" bd: %p\n", bd
);
1429 printf("&bde: %p\n", bde
);
1430 printf("*bde: %lx\n", *(unsigned long *)bde
);
1431 if (!bd_entry_valid(bde
))
1434 bte
= mpx_vaddr_to_bt_entry(buf
, bd
);
1435 printf(" te: %p\n", bte
);
1436 printf("bte[0]: %lx\n", bte
->contents
[0]);
1437 printf("bte[1]: %lx\n", bte
->contents
[1]);
1438 printf("bte[2]: %lx\n", bte
->contents
[2]);
1439 printf("bte[3]: %lx\n", bte
->contents
[3]);
1445 void check_mpx_insns_and_tables(void)
1449 int buf_size
= (1024*1024);
1450 unsigned long *buf
= malloc(buf_size
);
1451 const int total_nr_tests
= NR_MPX_TEST_FUNCTIONS
* TEST_ROUNDS
;
1454 memset(buf
, 0, buf_size
);
1455 memset(buf_shadow
, 0, sizeof(buf_shadow
));
1457 for (i
= 0; i
< TEST_ROUNDS
; i
++) {
1458 uint8_t *ptr
= get_random_addr() + 8;
1460 for (j
= 0; j
< NR_MPX_TEST_FUNCTIONS
; j
++) {
1465 dprintf2("starting test %d round %d\n", j
, i
);
1466 dprint_context(xsave_test_buf
);
1468 * test5 loads an address from the bounds tables.
1469 * The load will only complete if 'ptr' matches
1470 * the load and the store, so with random addrs,
1471 * the odds of this are very small. Make it
1472 * higher by only moving 'ptr' 1/10 times.
1474 if (random() % 10 <= 0)
1475 ptr
= get_random_addr() + 8;
1476 dprintf3("random ptr{%p}\n", ptr
);
1477 dprint_context(xsave_test_buf
);
1478 run_helpers(j
, (void *)buf
, (void *)buf_shadow
, ptr
);
1479 dprint_context(xsave_test_buf
);
1480 if (!compare_context(xsave_test_buf
)) {
1481 insn_test_failed(j
, i
, buf
, buf_shadow
, ptr
);
1486 dprint_context(xsave_test_buf
);
1487 dprintf2("finished test %d round %d\n", j
, i
);
1489 dprint_context(xsave_test_buf
);
1494 dprintf2("\nabout to free:\n");
1496 dprintf1("successes: %d\n", successes
);
1497 dprintf1(" failures: %d\n", failures
);
1498 dprintf1(" tests: %d\n", total_nr_tests
);
1499 dprintf1(" expected: %jd #BRs\n", num_upper_brs
+ num_lower_brs
);
1500 dprintf1(" saw: %d #BRs\n", br_count
);
1502 eprintf("ERROR: non-zero number of failures\n");
1505 if (successes
!= total_nr_tests
) {
1506 eprintf("ERROR: succeded fewer than number of tries (%d != %d)\n",
1507 successes
, total_nr_tests
);
1510 if (num_upper_brs
+ num_lower_brs
!= br_count
) {
1511 eprintf("ERROR: unexpected number of #BRs: %jd %jd %d\n",
1512 num_upper_brs
, num_lower_brs
, br_count
);
1513 eprintf("successes: %d\n", successes
);
1514 eprintf(" failures: %d\n", failures
);
1515 eprintf(" tests: %d\n", total_nr_tests
);
1516 eprintf(" expected: %jd #BRs\n", num_upper_brs
+ num_lower_brs
);
1517 eprintf(" saw: %d #BRs\n", br_count
);
1523 * This is supposed to SIGSEGV nicely once the kernel
1524 * can no longer allocate vaddr space.
1526 void exhaust_vaddr_space(void)
1529 /* Try to make sure there is no room for a bounds table anywhere */
1530 unsigned long skip
= MPX_BOUNDS_TABLE_SIZE_BYTES
- PAGE_SIZE
;
1532 unsigned long max_vaddr
= 0xf7788000UL
;
1534 unsigned long max_vaddr
= 0x800000000000UL
;
1537 dprintf1("%s() start\n", __func__
);
1538 /* do not start at 0, we aren't allowed to map there */
1539 for (ptr
= PAGE_SIZE
; ptr
< max_vaddr
; ptr
+= skip
) {
1541 int ret
= madvise((void *)ptr
, PAGE_SIZE
, MADV_NORMAL
);
1544 dprintf1("madvise() %lx ret: %d\n", ptr
, ret
);
1547 ptr_ret
= mmap((void *)ptr
, PAGE_SIZE
, PROT_READ
|PROT_WRITE
,
1548 MAP_ANONYMOUS
|MAP_PRIVATE
, -1, 0);
1549 if (ptr_ret
!= (void *)ptr
) {
1551 dprintf1("mmap(%lx) ret: %p\n", ptr
, ptr_ret
);
1554 if (!(ptr
& 0xffffff))
1555 dprintf1("mmap(%lx) ret: %p\n", ptr
, ptr_ret
);
1557 for (ptr
= PAGE_SIZE
; ptr
< max_vaddr
; ptr
+= skip
) {
1558 dprintf2("covering 0x%lx with bounds table entries\n", ptr
);
1559 cover_buf_with_bt_entries((void *)ptr
, PAGE_SIZE
);
1561 dprintf1("%s() end\n", __func__
);
1562 printf("done with vaddr space fun\n");
1565 void mpx_table_test(void)
1567 printf("starting mpx bounds table test\n");
1568 run_timed_test(check_mpx_insns_and_tables
);
1569 printf("done with mpx bounds table test\n");
1572 int main(int argc
, char **argv
)
1575 int vaddrexhaust
= 0;
1579 check_mpx_support();
1589 xsave_state((void *)xsave_test_buf
, 0x1f);
1590 if (!compare_context(xsave_test_buf
))
1591 printf("Init failed\n");
1593 for (i
= 1; i
< argc
; i
++) {
1594 if (!strcmp(argv
[i
], "unmaptest"))
1596 if (!strcmp(argv
[i
], "vaddrexhaust"))
1598 if (!strcmp(argv
[i
], "tabletest"))
1601 if (!(unmaptest
|| vaddrexhaust
|| tabletest
)) {
1603 /* vaddrexhaust = 1; */
1607 check_bounds_table_frees();
1611 exhaust_vaddr_space();
1612 printf("%s completed successfully\n", argv
[0]);
1616 #include "mpx-dig.c"