1 // SPDX-License-Identifier: GPL-2.0
3 * Tests x86 Memory Protection Keys (see Documentation/x86/protection-keys.txt)
5 * There are examples in here of:
6 * * how to set protection keys on memory
7 * * how to set/clear bits in PKRU (the rights register)
8 * * how to handle SEGV_PKRU signals and extract pkey-relevant
9 * information from the siginfo
12 * make sure KSM and KSM COW breaking works
13 * prefault pages in at malloc, or not
14 * protect MPX bounds tables with protection keys?
15 * make sure VMA splitting/merging is working correctly
16 * OOMs can destroy mm->mmap (see exit_mmap()), so make sure it is immune to pkeys
17 * look for pkey "leaks" where it is still set on a VMA but "freed" back to the kernel
18 * do a plain mprotect() to a mprotect_pkey() area and make sure the pkey sticks
21 * gcc -o protection_keys -O2 -g -std=gnu99 -pthread -Wall protection_keys.c -lrt -ldl -lm
22 * gcc -m32 -o protection_keys_32 -O2 -g -std=gnu99 -pthread -Wall protection_keys.c -lrt -ldl -lm
26 #include <linux/futex.h>
28 #include <sys/syscall.h>
38 #include <sys/types.h>
43 #include <sys/ptrace.h>
46 #include "pkey-helpers.h"
51 unsigned int shadow_pkru
;
53 #define HPAGE_SIZE (1UL<<21)
54 #define ARRAY_SIZE(x) (sizeof(x) / sizeof(*(x)))
55 #define ALIGN_UP(x, align_to) (((x) + ((align_to)-1)) & ~((align_to)-1))
56 #define ALIGN_DOWN(x, align_to) ((x) & ~((align_to)-1))
57 #define ALIGN_PTR_UP(p, ptr_align_to) ((typeof(p))ALIGN_UP((unsigned long)(p), ptr_align_to))
58 #define ALIGN_PTR_DOWN(p, ptr_align_to) ((typeof(p))ALIGN_DOWN((unsigned long)(p), ptr_align_to))
59 #define __stringify_1(x...) #x
60 #define __stringify(x...) __stringify_1(x)
62 #define PTR_ERR_ENOTSUP ((void *)-ENOTSUP)
65 char dprint_in_signal_buffer
[DPRINT_IN_SIGNAL_BUF_SIZE
];
67 extern void abort_hooks(void);
68 #define pkey_assert(condition) do { \
70 dprintf0("assert() at %s::%d test_nr: %d iteration: %d\n", \
72 test_nr, iteration_nr); \
73 dprintf0("errno at assert: %d", errno); \
79 void cat_into_file(char *str
, char *file
)
81 int fd
= open(file
, O_RDWR
);
84 dprintf2("%s(): writing '%s' to '%s'\n", __func__
, str
, file
);
86 * these need to be raw because they are called under
90 fprintf(stderr
, "error opening '%s'\n", str
);
95 ret
= write(fd
, str
, strlen(str
));
96 if (ret
!= strlen(str
)) {
97 perror("write to file failed");
98 fprintf(stderr
, "filename: '%s' str: '%s'\n", file
, str
);
104 #if CONTROL_TRACING > 0
105 static int warned_tracing
;
106 int tracing_root_ok(void)
108 if (geteuid() != 0) {
110 fprintf(stderr
, "WARNING: not run as root, "
111 "can not do tracing control\n");
119 void tracing_on(void)
121 #if CONTROL_TRACING > 0
122 #define TRACEDIR "/sys/kernel/debug/tracing"
125 if (!tracing_root_ok())
128 sprintf(pidstr
, "%d", getpid());
129 cat_into_file("0", TRACEDIR
"/tracing_on");
130 cat_into_file("\n", TRACEDIR
"/trace");
132 cat_into_file("function_graph", TRACEDIR
"/current_tracer");
133 cat_into_file("1", TRACEDIR
"/options/funcgraph-proc");
135 cat_into_file("nop", TRACEDIR
"/current_tracer");
137 cat_into_file(pidstr
, TRACEDIR
"/set_ftrace_pid");
138 cat_into_file("1", TRACEDIR
"/tracing_on");
139 dprintf1("enabled tracing\n");
143 void tracing_off(void)
145 #if CONTROL_TRACING > 0
146 if (!tracing_root_ok())
148 cat_into_file("0", "/sys/kernel/debug/tracing/tracing_on");
152 void abort_hooks(void)
154 fprintf(stderr
, "running %s()...\n", __func__
);
156 #ifdef SLEEP_ON_ABORT
157 sleep(SLEEP_ON_ABORT
);
161 static inline void __page_o_noops(void)
163 /* 8-bytes of instruction * 512 bytes = 1 page */
164 asm(".rept 512 ; nopl 0x7eeeeeee(%eax) ; .endr");
168 * This attempts to have roughly a page of instructions followed by a few
169 * instructions that do a write, and another page of instructions. That
170 * way, we are pretty sure that the write is in the second page of
171 * instructions and has at least a page of padding behind it.
173 * *That* lets us be sure to madvise() away the write instruction, which
174 * will then fault, which makes sure that the fault code handles
175 * execute-only memory properly.
177 __attribute__((__aligned__(PAGE_SIZE
)))
178 void lots_o_noops_around_write(int *write_to_me
)
180 dprintf3("running %s()\n", __func__
);
182 /* Assume this happens in the second page of instructions: */
183 *write_to_me
= __LINE__
;
184 /* pad out by another page: */
186 dprintf3("%s() done\n", __func__
);
189 /* Define some kernel-like types */
197 #ifndef SYS_mprotect_key
198 # define SYS_mprotect_key 380
201 #ifndef SYS_pkey_alloc
202 # define SYS_pkey_alloc 381
203 # define SYS_pkey_free 382
206 #define REG_IP_IDX REG_EIP
207 #define si_pkey_offset 0x14
211 #ifndef SYS_mprotect_key
212 # define SYS_mprotect_key 329
215 #ifndef SYS_pkey_alloc
216 # define SYS_pkey_alloc 330
217 # define SYS_pkey_free 331
220 #define REG_IP_IDX REG_RIP
221 #define si_pkey_offset 0x20
225 void dump_mem(void *dumpme
, int len_bytes
)
227 char *c
= (void *)dumpme
;
230 for (i
= 0; i
< len_bytes
; i
+= sizeof(u64
)) {
231 u64
*ptr
= (u64
*)(c
+ i
);
232 dprintf1("dump[%03d][@%p]: %016jx\n", i
, ptr
, *ptr
);
236 /* Failed address bound checks: */
238 # define SEGV_BNDERR 3
242 # define SEGV_PKUERR 4
245 static char *si_code_str(int si_code
)
247 if (si_code
== SEGV_MAPERR
)
248 return "SEGV_MAPERR";
249 if (si_code
== SEGV_ACCERR
)
250 return "SEGV_ACCERR";
251 if (si_code
== SEGV_BNDERR
)
252 return "SEGV_BNDERR";
253 if (si_code
== SEGV_PKUERR
)
254 return "SEGV_PKUERR";
259 int last_si_pkey
= -1;
260 void signal_handler(int signum
, siginfo_t
*si
, void *vucontext
)
262 ucontext_t
*uctxt
= vucontext
;
272 dprint_in_signal
= 1;
273 dprintf1(">>>>===============SIGSEGV============================\n");
274 dprintf1("%s()::%d, pkru: 0x%x shadow: %x\n", __func__
, __LINE__
,
275 __rdpkru(), shadow_pkru
);
277 trapno
= uctxt
->uc_mcontext
.gregs
[REG_TRAPNO
];
278 ip
= uctxt
->uc_mcontext
.gregs
[REG_IP_IDX
];
279 fpregset
= uctxt
->uc_mcontext
.fpregs
;
280 fpregs
= (void *)fpregset
;
282 dprintf2("%s() trapno: %d ip: 0x%lx info->si_code: %s/%d\n", __func__
,
283 trapno
, ip
, si_code_str(si
->si_code
), si
->si_code
);
286 * 32-bit has some extra padding so that userspace can tell whether
287 * the XSTATE header is present in addition to the "legacy" FPU
288 * state. We just assume that it is here.
292 pkru_offset
= pkru_xstate_offset();
293 pkru_ptr
= (void *)(&fpregs
[pkru_offset
]);
295 dprintf1("siginfo: %p\n", si
);
296 dprintf1(" fpregs: %p\n", fpregs
);
298 * If we got a PKRU fault, we *HAVE* to have at least one bit set in
301 dprintf1("pkru_xstate_offset: %d\n", pkru_xstate_offset());
303 dump_mem(pkru_ptr
- 128, 256);
304 pkey_assert(*pkru_ptr
);
306 si_pkey_ptr
= (u32
*)(((u8
*)si
) + si_pkey_offset
);
307 dprintf1("si_pkey_ptr: %p\n", si_pkey_ptr
);
308 dump_mem(si_pkey_ptr
- 8, 24);
309 siginfo_pkey
= *si_pkey_ptr
;
310 pkey_assert(siginfo_pkey
< NR_PKEYS
);
311 last_si_pkey
= siginfo_pkey
;
313 if ((si
->si_code
== SEGV_MAPERR
) ||
314 (si
->si_code
== SEGV_ACCERR
) ||
315 (si
->si_code
== SEGV_BNDERR
)) {
316 printf("non-PK si_code, exiting...\n");
320 dprintf1("signal pkru from xsave: %08x\n", *pkru_ptr
);
321 /* need __rdpkru() version so we do not do shadow_pkru checking */
322 dprintf1("signal pkru from pkru: %08x\n", __rdpkru());
323 dprintf1("pkey from siginfo: %jx\n", siginfo_pkey
);
324 *(u64
*)pkru_ptr
= 0x00000000;
325 dprintf1("WARNING: set PRKU=0 to allow faulting instruction to continue\n");
327 dprintf1("<<<<==================================================\n");
328 dprint_in_signal
= 0;
331 int wait_all_children(void)
334 return waitpid(-1, &status
, 0);
339 dprint_in_signal
= 1;
340 dprintf2("[%d] SIGCHLD: %d\n", getpid(), x
);
341 dprint_in_signal
= 0;
344 void setup_sigsegv_handler(void)
347 struct sigaction newact
;
348 struct sigaction oldact
;
350 /* #PF is mapped to sigsegv */
351 int signum
= SIGSEGV
;
353 newact
.sa_handler
= 0;
354 newact
.sa_sigaction
= signal_handler
;
356 /*sigset_t - signals to block while in the handler */
357 /* get the old signal mask. */
358 rs
= sigprocmask(SIG_SETMASK
, 0, &newact
.sa_mask
);
359 pkey_assert(rs
== 0);
361 /* call sa_sigaction, not sa_handler*/
362 newact
.sa_flags
= SA_SIGINFO
;
364 newact
.sa_restorer
= 0; /* void(*)(), obsolete */
365 r
= sigaction(signum
, &newact
, &oldact
);
366 r
= sigaction(SIGALRM
, &newact
, &oldact
);
370 void setup_handlers(void)
372 signal(SIGCHLD
, &sig_chld
);
373 setup_sigsegv_handler();
376 pid_t
fork_lazy_child(void)
381 pkey_assert(forkret
>= 0);
382 dprintf3("[%d] fork() ret: %d\n", getpid(), forkret
);
387 dprintf1("child sleeping...\n");
394 #ifndef PKEY_DISABLE_ACCESS
395 # define PKEY_DISABLE_ACCESS 0x1
398 #ifndef PKEY_DISABLE_WRITE
399 # define PKEY_DISABLE_WRITE 0x2
402 static u32
hw_pkey_get(int pkey
, unsigned long flags
)
404 u32 mask
= (PKEY_DISABLE_ACCESS
|PKEY_DISABLE_WRITE
);
405 u32 pkru
= __rdpkru();
409 dprintf1("%s(pkey=%d, flags=%lx) = %x / %d\n",
410 __func__
, pkey
, flags
, 0, 0);
411 dprintf2("%s() raw pkru: %x\n", __func__
, pkru
);
413 shifted_pkru
= (pkru
>> (pkey
* PKRU_BITS_PER_PKEY
));
414 dprintf2("%s() shifted_pkru: %x\n", __func__
, shifted_pkru
);
415 masked_pkru
= shifted_pkru
& mask
;
416 dprintf2("%s() masked pkru: %x\n", __func__
, masked_pkru
);
418 * shift down the relevant bits to the lowest two, then
419 * mask off all the other high bits.
424 static int hw_pkey_set(int pkey
, unsigned long rights
, unsigned long flags
)
426 u32 mask
= (PKEY_DISABLE_ACCESS
|PKEY_DISABLE_WRITE
);
427 u32 old_pkru
= __rdpkru();
430 /* make sure that 'rights' only contains the bits we expect: */
431 assert(!(rights
& ~mask
));
435 /* mask out bits from pkey in old value: */
436 new_pkru
&= ~(mask
<< (pkey
* PKRU_BITS_PER_PKEY
));
437 /* OR in new bits for pkey: */
438 new_pkru
|= (rights
<< (pkey
* PKRU_BITS_PER_PKEY
));
442 dprintf3("%s(pkey=%d, rights=%lx, flags=%lx) = %x pkru now: %x old_pkru: %x\n",
443 __func__
, pkey
, rights
, flags
, 0, __rdpkru(), old_pkru
);
447 void pkey_disable_set(int pkey
, int flags
)
449 unsigned long syscall_flags
= 0;
452 u32 orig_pkru
= rdpkru();
454 dprintf1("START->%s(%d, 0x%x)\n", __func__
,
456 pkey_assert(flags
& (PKEY_DISABLE_ACCESS
| PKEY_DISABLE_WRITE
));
458 pkey_rights
= hw_pkey_get(pkey
, syscall_flags
);
460 dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__
,
461 pkey
, pkey
, pkey_rights
);
462 pkey_assert(pkey_rights
>= 0);
464 pkey_rights
|= flags
;
466 ret
= hw_pkey_set(pkey
, pkey_rights
, syscall_flags
);
468 /*pkru and flags have the same format */
469 shadow_pkru
|= flags
<< (pkey
* 2);
470 dprintf1("%s(%d) shadow: 0x%x\n", __func__
, pkey
, shadow_pkru
);
472 pkey_assert(ret
>= 0);
474 pkey_rights
= hw_pkey_get(pkey
, syscall_flags
);
475 dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__
,
476 pkey
, pkey
, pkey_rights
);
478 dprintf1("%s(%d) pkru: 0x%x\n", __func__
, pkey
, rdpkru());
480 pkey_assert(rdpkru() > orig_pkru
);
481 dprintf1("END<---%s(%d, 0x%x)\n", __func__
,
485 void pkey_disable_clear(int pkey
, int flags
)
487 unsigned long syscall_flags
= 0;
489 int pkey_rights
= hw_pkey_get(pkey
, syscall_flags
);
490 u32 orig_pkru
= rdpkru();
492 pkey_assert(flags
& (PKEY_DISABLE_ACCESS
| PKEY_DISABLE_WRITE
));
494 dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__
,
495 pkey
, pkey
, pkey_rights
);
496 pkey_assert(pkey_rights
>= 0);
498 pkey_rights
|= flags
;
500 ret
= hw_pkey_set(pkey
, pkey_rights
, 0);
501 /* pkru and flags have the same format */
502 shadow_pkru
&= ~(flags
<< (pkey
* 2));
503 pkey_assert(ret
>= 0);
505 pkey_rights
= hw_pkey_get(pkey
, syscall_flags
);
506 dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__
,
507 pkey
, pkey
, pkey_rights
);
509 dprintf1("%s(%d) pkru: 0x%x\n", __func__
, pkey
, rdpkru());
511 assert(rdpkru() > orig_pkru
);
514 void pkey_write_allow(int pkey
)
516 pkey_disable_clear(pkey
, PKEY_DISABLE_WRITE
);
518 void pkey_write_deny(int pkey
)
520 pkey_disable_set(pkey
, PKEY_DISABLE_WRITE
);
522 void pkey_access_allow(int pkey
)
524 pkey_disable_clear(pkey
, PKEY_DISABLE_ACCESS
);
526 void pkey_access_deny(int pkey
)
528 pkey_disable_set(pkey
, PKEY_DISABLE_ACCESS
);
531 int sys_mprotect_pkey(void *ptr
, size_t size
, unsigned long orig_prot
,
536 dprintf2("%s(0x%p, %zx, prot=%lx, pkey=%lx)\n", __func__
,
537 ptr
, size
, orig_prot
, pkey
);
540 sret
= syscall(SYS_mprotect_key
, ptr
, size
, orig_prot
, pkey
);
542 dprintf2("SYS_mprotect_key sret: %d\n", sret
);
543 dprintf2("SYS_mprotect_key prot: 0x%lx\n", orig_prot
);
544 dprintf2("SYS_mprotect_key failed, errno: %d\n", errno
);
545 if (DEBUG_LEVEL
>= 2)
546 perror("SYS_mprotect_pkey");
551 int sys_pkey_alloc(unsigned long flags
, unsigned long init_val
)
553 int ret
= syscall(SYS_pkey_alloc
, flags
, init_val
);
554 dprintf1("%s(flags=%lx, init_val=%lx) syscall ret: %d errno: %d\n",
555 __func__
, flags
, init_val
, ret
, errno
);
562 unsigned long init_val
= 0x0;
564 dprintf1("alloc_pkey()::%d, pkru: 0x%x shadow: %x\n",
565 __LINE__
, __rdpkru(), shadow_pkru
);
566 ret
= sys_pkey_alloc(0, init_val
);
568 * pkey_alloc() sets PKRU, so we need to reflect it in
571 dprintf4("alloc_pkey()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n",
572 __LINE__
, ret
, __rdpkru(), shadow_pkru
);
574 /* clear both the bits: */
575 shadow_pkru
&= ~(0x3 << (ret
* 2));
576 dprintf4("alloc_pkey()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n",
577 __LINE__
, ret
, __rdpkru(), shadow_pkru
);
579 * move the new state in from init_val
580 * (remember, we cheated and init_val == pkru format)
582 shadow_pkru
|= (init_val
<< (ret
* 2));
584 dprintf4("alloc_pkey()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n",
585 __LINE__
, ret
, __rdpkru(), shadow_pkru
);
586 dprintf1("alloc_pkey()::%d errno: %d\n", __LINE__
, errno
);
587 /* for shadow checking: */
589 dprintf4("alloc_pkey()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n",
590 __LINE__
, ret
, __rdpkru(), shadow_pkru
);
594 int sys_pkey_free(unsigned long pkey
)
596 int ret
= syscall(SYS_pkey_free
, pkey
);
597 dprintf1("%s(pkey=%ld) syscall ret: %d\n", __func__
, pkey
, ret
);
602 * I had a bug where pkey bits could be set by mprotect() but
603 * not cleared. This ensures we get lots of random bit sets
604 * and clears on the vma and pte pkey bits.
606 int alloc_random_pkey(void)
608 int max_nr_pkey_allocs
;
611 int alloced_pkeys
[NR_PKEYS
];
614 memset(alloced_pkeys
, 0, sizeof(alloced_pkeys
));
616 /* allocate every possible key and make a note of which ones we got */
617 max_nr_pkey_allocs
= NR_PKEYS
;
618 max_nr_pkey_allocs
= 1;
619 for (i
= 0; i
< max_nr_pkey_allocs
; i
++) {
620 int new_pkey
= alloc_pkey();
623 alloced_pkeys
[nr_alloced
++] = new_pkey
;
626 pkey_assert(nr_alloced
> 0);
627 /* select a random one out of the allocated ones */
628 random_index
= rand() % nr_alloced
;
629 ret
= alloced_pkeys
[random_index
];
630 /* now zero it out so we don't free it next */
631 alloced_pkeys
[random_index
] = 0;
633 /* go through the allocated ones that we did not want and free them */
634 for (i
= 0; i
< nr_alloced
; i
++) {
636 if (!alloced_pkeys
[i
])
638 free_ret
= sys_pkey_free(alloced_pkeys
[i
]);
639 pkey_assert(!free_ret
);
641 dprintf1("%s()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", __func__
,
642 __LINE__
, ret
, __rdpkru(), shadow_pkru
);
646 int mprotect_pkey(void *ptr
, size_t size
, unsigned long orig_prot
,
649 int nr_iterations
= random() % 100;
653 int rpkey
= alloc_random_pkey();
654 ret
= sys_mprotect_pkey(ptr
, size
, orig_prot
, pkey
);
655 dprintf1("sys_mprotect_pkey(%p, %zx, prot=0x%lx, pkey=%ld) ret: %d\n",
656 ptr
, size
, orig_prot
, pkey
, ret
);
657 if (nr_iterations
-- < 0)
660 dprintf1("%s()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", __func__
,
661 __LINE__
, ret
, __rdpkru(), shadow_pkru
);
662 sys_pkey_free(rpkey
);
663 dprintf1("%s()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", __func__
,
664 __LINE__
, ret
, __rdpkru(), shadow_pkru
);
666 pkey_assert(pkey
< NR_PKEYS
);
668 ret
= sys_mprotect_pkey(ptr
, size
, orig_prot
, pkey
);
669 dprintf1("mprotect_pkey(%p, %zx, prot=0x%lx, pkey=%ld) ret: %d\n",
670 ptr
, size
, orig_prot
, pkey
, ret
);
672 dprintf1("%s()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", __func__
,
673 __LINE__
, ret
, __rdpkru(), shadow_pkru
);
677 struct pkey_malloc_record
{
681 struct pkey_malloc_record
*pkey_malloc_records
;
682 long nr_pkey_malloc_records
;
683 void record_pkey_malloc(void *ptr
, long size
)
686 struct pkey_malloc_record
*rec
= NULL
;
688 for (i
= 0; i
< nr_pkey_malloc_records
; i
++) {
689 rec
= &pkey_malloc_records
[i
];
690 /* find a free record */
695 /* every record is full */
696 size_t old_nr_records
= nr_pkey_malloc_records
;
697 size_t new_nr_records
= (nr_pkey_malloc_records
* 2 + 1);
698 size_t new_size
= new_nr_records
* sizeof(struct pkey_malloc_record
);
699 dprintf2("new_nr_records: %zd\n", new_nr_records
);
700 dprintf2("new_size: %zd\n", new_size
);
701 pkey_malloc_records
= realloc(pkey_malloc_records
, new_size
);
702 pkey_assert(pkey_malloc_records
!= NULL
);
703 rec
= &pkey_malloc_records
[nr_pkey_malloc_records
];
705 * realloc() does not initialize memory, so zero it from
706 * the first new record all the way to the end.
708 for (i
= 0; i
< new_nr_records
- old_nr_records
; i
++)
709 memset(rec
+ i
, 0, sizeof(*rec
));
711 dprintf3("filling malloc record[%d/%p]: {%p, %ld}\n",
712 (int)(rec
- pkey_malloc_records
), rec
, ptr
, size
);
715 nr_pkey_malloc_records
++;
718 void free_pkey_malloc(void *ptr
)
722 dprintf3("%s(%p)\n", __func__
, ptr
);
723 for (i
= 0; i
< nr_pkey_malloc_records
; i
++) {
724 struct pkey_malloc_record
*rec
= &pkey_malloc_records
[i
];
725 dprintf4("looking for ptr %p at record[%ld/%p]: {%p, %ld}\n",
726 ptr
, i
, rec
, rec
->ptr
, rec
->size
);
727 if ((ptr
< rec
->ptr
) ||
728 (ptr
>= rec
->ptr
+ rec
->size
))
731 dprintf3("found ptr %p at record[%ld/%p]: {%p, %ld}\n",
732 ptr
, i
, rec
, rec
->ptr
, rec
->size
);
733 nr_pkey_malloc_records
--;
734 ret
= munmap(rec
->ptr
, rec
->size
);
735 dprintf3("munmap ret: %d\n", ret
);
737 dprintf3("clearing rec->ptr, rec: %p\n", rec
);
739 dprintf3("done clearing rec->ptr, rec: %p\n", rec
);
746 void *malloc_pkey_with_mprotect(long size
, int prot
, u16 pkey
)
752 dprintf1("doing %s(size=%ld, prot=0x%x, pkey=%d)\n", __func__
,
754 pkey_assert(pkey
< NR_PKEYS
);
755 ptr
= mmap(NULL
, size
, prot
, MAP_ANONYMOUS
|MAP_PRIVATE
, -1, 0);
756 pkey_assert(ptr
!= (void *)-1);
757 ret
= mprotect_pkey((void *)ptr
, PAGE_SIZE
, prot
, pkey
);
759 record_pkey_malloc(ptr
, size
);
762 dprintf1("%s() for pkey %d @ %p\n", __func__
, pkey
, ptr
);
766 void *malloc_pkey_anon_huge(long size
, int prot
, u16 pkey
)
771 dprintf1("doing %s(size=%ld, prot=0x%x, pkey=%d)\n", __func__
,
774 * Guarantee we can fit at least one huge page in the resulting
775 * allocation by allocating space for 2:
777 size
= ALIGN_UP(size
, HPAGE_SIZE
* 2);
778 ptr
= mmap(NULL
, size
, PROT_NONE
, MAP_ANONYMOUS
|MAP_PRIVATE
, -1, 0);
779 pkey_assert(ptr
!= (void *)-1);
780 record_pkey_malloc(ptr
, size
);
781 mprotect_pkey(ptr
, size
, prot
, pkey
);
783 dprintf1("unaligned ptr: %p\n", ptr
);
784 ptr
= ALIGN_PTR_UP(ptr
, HPAGE_SIZE
);
785 dprintf1(" aligned ptr: %p\n", ptr
);
786 ret
= madvise(ptr
, HPAGE_SIZE
, MADV_HUGEPAGE
);
787 dprintf1("MADV_HUGEPAGE ret: %d\n", ret
);
788 ret
= madvise(ptr
, HPAGE_SIZE
, MADV_WILLNEED
);
789 dprintf1("MADV_WILLNEED ret: %d\n", ret
);
790 memset(ptr
, 0, HPAGE_SIZE
);
792 dprintf1("mmap()'d thp for pkey %d @ %p\n", pkey
, ptr
);
796 int hugetlb_setup_ok
;
797 #define GET_NR_HUGE_PAGES 10
798 void setup_hugetlbfs(void)
804 if (geteuid() != 0) {
805 fprintf(stderr
, "WARNING: not run as root, can not do hugetlb test\n");
809 cat_into_file(__stringify(GET_NR_HUGE_PAGES
), "/proc/sys/vm/nr_hugepages");
812 * Now go make sure that we got the pages and that they
813 * are 2M pages. Someone might have made 1G the default.
815 fd
= open("/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages", O_RDONLY
);
817 perror("opening sysfs 2M hugetlb config");
821 /* -1 to guarantee leaving the trailing \0 */
822 err
= read(fd
, buf
, sizeof(buf
)-1);
825 perror("reading sysfs 2M hugetlb config");
829 if (atoi(buf
) != GET_NR_HUGE_PAGES
) {
830 fprintf(stderr
, "could not confirm 2M pages, got: '%s' expected %d\n",
831 buf
, GET_NR_HUGE_PAGES
);
835 hugetlb_setup_ok
= 1;
838 void *malloc_pkey_hugetlb(long size
, int prot
, u16 pkey
)
841 int flags
= MAP_ANONYMOUS
|MAP_PRIVATE
|MAP_HUGETLB
;
843 if (!hugetlb_setup_ok
)
844 return PTR_ERR_ENOTSUP
;
846 dprintf1("doing %s(%ld, %x, %x)\n", __func__
, size
, prot
, pkey
);
847 size
= ALIGN_UP(size
, HPAGE_SIZE
* 2);
848 pkey_assert(pkey
< NR_PKEYS
);
849 ptr
= mmap(NULL
, size
, PROT_NONE
, flags
, -1, 0);
850 pkey_assert(ptr
!= (void *)-1);
851 mprotect_pkey(ptr
, size
, prot
, pkey
);
853 record_pkey_malloc(ptr
, size
);
855 dprintf1("mmap()'d hugetlbfs for pkey %d @ %p\n", pkey
, ptr
);
859 void *malloc_pkey_mmap_dax(long size
, int prot
, u16 pkey
)
864 dprintf1("doing %s(size=%ld, prot=0x%x, pkey=%d)\n", __func__
,
866 pkey_assert(pkey
< NR_PKEYS
);
867 fd
= open("/dax/foo", O_RDWR
);
868 pkey_assert(fd
>= 0);
870 ptr
= mmap(0, size
, prot
, MAP_SHARED
, fd
, 0);
871 pkey_assert(ptr
!= (void *)-1);
873 mprotect_pkey(ptr
, size
, prot
, pkey
);
875 record_pkey_malloc(ptr
, size
);
877 dprintf1("mmap()'d for pkey %d @ %p\n", pkey
, ptr
);
882 void *(*pkey_malloc
[])(long size
, int prot
, u16 pkey
) = {
884 malloc_pkey_with_mprotect
,
885 malloc_pkey_anon_huge
,
887 /* can not do direct with the pkey_mprotect() API:
888 malloc_pkey_mmap_direct,
889 malloc_pkey_mmap_dax,
893 void *malloc_pkey(long size
, int prot
, u16 pkey
)
896 static int malloc_type
;
897 int nr_malloc_types
= ARRAY_SIZE(pkey_malloc
);
899 pkey_assert(pkey
< NR_PKEYS
);
902 pkey_assert(malloc_type
< nr_malloc_types
);
904 ret
= pkey_malloc
[malloc_type
](size
, prot
, pkey
);
905 pkey_assert(ret
!= (void *)-1);
908 if (malloc_type
>= nr_malloc_types
)
909 malloc_type
= (random()%nr_malloc_types
);
911 /* try again if the malloc_type we tried is unsupported */
912 if (ret
== PTR_ERR_ENOTSUP
)
918 dprintf3("%s(%ld, prot=%x, pkey=%x) returning: %p\n", __func__
,
919 size
, prot
, pkey
, ret
);
923 int last_pkru_faults
;
924 #define UNKNOWN_PKEY -2
925 void expected_pk_fault(int pkey
)
927 dprintf2("%s(): last_pkru_faults: %d pkru_faults: %d\n",
928 __func__
, last_pkru_faults
, pkru_faults
);
929 dprintf2("%s(%d): last_si_pkey: %d\n", __func__
, pkey
, last_si_pkey
);
930 pkey_assert(last_pkru_faults
+ 1 == pkru_faults
);
933 * For exec-only memory, we do not know the pkey in
934 * advance, so skip this check.
936 if (pkey
!= UNKNOWN_PKEY
)
937 pkey_assert(last_si_pkey
== pkey
);
940 * The signal handler shold have cleared out PKRU to let the
941 * test program continue. We now have to restore it.
946 __wrpkru(shadow_pkru
);
947 dprintf1("%s() set PKRU=%x to restore state after signal nuked it\n",
948 __func__
, shadow_pkru
);
949 last_pkru_faults
= pkru_faults
;
953 #define do_not_expect_pk_fault(msg) do { \
954 if (last_pkru_faults != pkru_faults) \
955 dprintf0("unexpected PK fault: %s\n", msg); \
956 pkey_assert(last_pkru_faults == pkru_faults); \
959 int test_fds
[10] = { -1 };
961 void __save_test_fd(int fd
)
963 pkey_assert(fd
>= 0);
964 pkey_assert(nr_test_fds
< ARRAY_SIZE(test_fds
));
965 test_fds
[nr_test_fds
] = fd
;
969 int get_test_read_fd(void)
971 int test_fd
= open("/etc/passwd", O_RDONLY
);
972 __save_test_fd(test_fd
);
976 void close_test_fds(void)
980 for (i
= 0; i
< nr_test_fds
; i
++) {
989 #define barrier() __asm__ __volatile__("": : :"memory")
990 __attribute__((noinline
)) int read_ptr(int *ptr
)
993 * Keep GCC from optimizing this away somehow
999 void test_read_of_write_disabled_region(int *ptr
, u16 pkey
)
1003 dprintf1("disabling write access to PKEY[1], doing read\n");
1004 pkey_write_deny(pkey
);
1005 ptr_contents
= read_ptr(ptr
);
1006 dprintf1("*ptr: %d\n", ptr_contents
);
1009 void test_read_of_access_disabled_region(int *ptr
, u16 pkey
)
1013 dprintf1("disabling access to PKEY[%02d], doing read @ %p\n", pkey
, ptr
);
1015 pkey_access_deny(pkey
);
1016 ptr_contents
= read_ptr(ptr
);
1017 dprintf1("*ptr: %d\n", ptr_contents
);
1018 expected_pk_fault(pkey
);
1020 void test_write_of_write_disabled_region(int *ptr
, u16 pkey
)
1022 dprintf1("disabling write access to PKEY[%02d], doing write\n", pkey
);
1023 pkey_write_deny(pkey
);
1025 expected_pk_fault(pkey
);
1027 void test_write_of_access_disabled_region(int *ptr
, u16 pkey
)
1029 dprintf1("disabling access to PKEY[%02d], doing write\n", pkey
);
1030 pkey_access_deny(pkey
);
1032 expected_pk_fault(pkey
);
1034 void test_kernel_write_of_access_disabled_region(int *ptr
, u16 pkey
)
1037 int test_fd
= get_test_read_fd();
1039 dprintf1("disabling access to PKEY[%02d], "
1040 "having kernel read() to buffer\n", pkey
);
1041 pkey_access_deny(pkey
);
1042 ret
= read(test_fd
, ptr
, 1);
1043 dprintf1("read ret: %d\n", ret
);
1046 void test_kernel_write_of_write_disabled_region(int *ptr
, u16 pkey
)
1049 int test_fd
= get_test_read_fd();
1051 pkey_write_deny(pkey
);
1052 ret
= read(test_fd
, ptr
, 100);
1053 dprintf1("read ret: %d\n", ret
);
1054 if (ret
< 0 && (DEBUG_LEVEL
> 0))
1055 perror("verbose read result (OK for this to be bad)");
1059 void test_kernel_gup_of_access_disabled_region(int *ptr
, u16 pkey
)
1061 int pipe_ret
, vmsplice_ret
;
1065 pipe_ret
= pipe(pipe_fds
);
1067 pkey_assert(pipe_ret
== 0);
1068 dprintf1("disabling access to PKEY[%02d], "
1069 "having kernel vmsplice from buffer\n", pkey
);
1070 pkey_access_deny(pkey
);
1072 iov
.iov_len
= PAGE_SIZE
;
1073 vmsplice_ret
= vmsplice(pipe_fds
[1], &iov
, 1, SPLICE_F_GIFT
);
1074 dprintf1("vmsplice() ret: %d\n", vmsplice_ret
);
1075 pkey_assert(vmsplice_ret
== -1);
1081 void test_kernel_gup_write_to_write_disabled_region(int *ptr
, u16 pkey
)
1083 int ignored
= 0xdada;
1085 int some_int
= __LINE__
;
1087 dprintf1("disabling write to PKEY[%02d], "
1088 "doing futex gunk in buffer\n", pkey
);
1090 pkey_write_deny(pkey
);
1091 futex_ret
= syscall(SYS_futex
, ptr
, FUTEX_WAIT
, some_int
-1, NULL
,
1093 if (DEBUG_LEVEL
> 0)
1095 dprintf1("futex() ret: %d\n", futex_ret
);
1098 /* Assumes that all pkeys other than 'pkey' are unallocated */
1099 void test_pkey_syscalls_on_non_allocated_pkey(int *ptr
, u16 pkey
)
1104 /* Note: 0 is the default pkey, so don't mess with it */
1105 for (i
= 1; i
< NR_PKEYS
; i
++) {
1109 dprintf1("trying get/set/free to non-allocated pkey: %2d\n", i
);
1110 err
= sys_pkey_free(i
);
1113 err
= sys_pkey_free(i
);
1116 err
= sys_mprotect_pkey(ptr
, PAGE_SIZE
, PROT_READ
, i
);
1121 /* Assumes that all pkeys other than 'pkey' are unallocated */
1122 void test_pkey_syscalls_bad_args(int *ptr
, u16 pkey
)
1125 int bad_pkey
= NR_PKEYS
+99;
1127 /* pass a known-invalid pkey in: */
1128 err
= sys_mprotect_pkey(ptr
, PAGE_SIZE
, PROT_READ
, bad_pkey
);
1132 /* Assumes that all pkeys other than 'pkey' are unallocated */
1133 void test_pkey_alloc_exhaust(int *ptr
, u16 pkey
)
1136 int allocated_pkeys
[NR_PKEYS
] = {0};
1137 int nr_allocated_pkeys
= 0;
1140 for (i
= 0; i
< NR_PKEYS
*2; i
++) {
1142 dprintf1("%s() alloc loop: %d\n", __func__
, i
);
1143 new_pkey
= alloc_pkey();
1144 dprintf4("%s()::%d, err: %d pkru: 0x%x shadow: 0x%x\n", __func__
,
1145 __LINE__
, err
, __rdpkru(), shadow_pkru
);
1146 rdpkru(); /* for shadow checking */
1147 dprintf2("%s() errno: %d ENOSPC: %d\n", __func__
, errno
, ENOSPC
);
1148 if ((new_pkey
== -1) && (errno
== ENOSPC
)) {
1149 dprintf2("%s() failed to allocate pkey after %d tries\n",
1150 __func__
, nr_allocated_pkeys
);
1153 pkey_assert(nr_allocated_pkeys
< NR_PKEYS
);
1154 allocated_pkeys
[nr_allocated_pkeys
++] = new_pkey
;
1157 dprintf3("%s()::%d\n", __func__
, __LINE__
);
1160 * ensure it did not reach the end of the loop without
1163 pkey_assert(i
< NR_PKEYS
*2);
1166 * There are 16 pkeys supported in hardware. Three are
1167 * allocated by the time we get here:
1168 * 1. The default key (0)
1169 * 2. One possibly consumed by an execute-only mapping.
1170 * 3. One allocated by the test code and passed in via
1171 * 'pkey' to this function.
1172 * Ensure that we can allocate at least another 13 (16-3).
1174 pkey_assert(i
>= NR_PKEYS
-3);
1176 for (i
= 0; i
< nr_allocated_pkeys
; i
++) {
1177 err
= sys_pkey_free(allocated_pkeys
[i
]);
1179 rdpkru(); /* for shadow checking */
1183 void test_ptrace_of_child(int *ptr
, u16 pkey
)
1185 __attribute__((__unused__
)) int peek_result
;
1191 * This is the "control" for our little expermient. Make sure
1192 * we can always access it when ptracing.
1194 int *plain_ptr_unaligned
= malloc(HPAGE_SIZE
);
1195 int *plain_ptr
= ALIGN_PTR_UP(plain_ptr_unaligned
, PAGE_SIZE
);
1198 * Fork a child which is an exact copy of this process, of course.
1199 * That means we can do all of our tests via ptrace() and then plain
1200 * memory access and ensure they work differently.
1202 child_pid
= fork_lazy_child();
1203 dprintf1("[%d] child pid: %d\n", getpid(), child_pid
);
1205 ret
= ptrace(PTRACE_ATTACH
, child_pid
, ignored
, ignored
);
1208 dprintf1("[%d] attach ret: %ld %d\n", getpid(), ret
, __LINE__
);
1209 pkey_assert(ret
!= -1);
1210 ret
= waitpid(child_pid
, &status
, WUNTRACED
);
1211 if ((ret
!= child_pid
) || !(WIFSTOPPED(status
))) {
1212 fprintf(stderr
, "weird waitpid result %ld stat %x\n",
1216 dprintf2("waitpid ret: %ld\n", ret
);
1217 dprintf2("waitpid status: %d\n", status
);
1219 pkey_access_deny(pkey
);
1220 pkey_write_deny(pkey
);
1222 /* Write access, untested for now:
1223 ret = ptrace(PTRACE_POKEDATA, child_pid, peek_at, data);
1224 pkey_assert(ret != -1);
1225 dprintf1("poke at %p: %ld\n", peek_at, ret);
1229 * Try to access the pkey-protected "ptr" via ptrace:
1231 ret
= ptrace(PTRACE_PEEKDATA
, child_pid
, ptr
, ignored
);
1232 /* expect it to work, without an error: */
1233 pkey_assert(ret
!= -1);
1234 /* Now access from the current task, and expect an exception: */
1235 peek_result
= read_ptr(ptr
);
1236 expected_pk_fault(pkey
);
1239 * Try to access the NON-pkey-protected "plain_ptr" via ptrace:
1241 ret
= ptrace(PTRACE_PEEKDATA
, child_pid
, plain_ptr
, ignored
);
1242 /* expect it to work, without an error: */
1243 pkey_assert(ret
!= -1);
1244 /* Now access from the current task, and expect NO exception: */
1245 peek_result
= read_ptr(plain_ptr
);
1246 do_not_expect_pk_fault("read plain pointer after ptrace");
1248 ret
= ptrace(PTRACE_DETACH
, child_pid
, ignored
, 0);
1249 pkey_assert(ret
!= -1);
1251 ret
= kill(child_pid
, SIGKILL
);
1252 pkey_assert(ret
!= -1);
1256 free(plain_ptr_unaligned
);
1259 void *get_pointer_to_instructions(void)
1263 p1
= ALIGN_PTR_UP(&lots_o_noops_around_write
, PAGE_SIZE
);
1264 dprintf3("&lots_o_noops: %p\n", &lots_o_noops_around_write
);
1265 /* lots_o_noops_around_write should be page-aligned already */
1266 assert(p1
== &lots_o_noops_around_write
);
1268 /* Point 'p1' at the *second* page of the function: */
1272 * Try to ensure we fault this in on next touch to ensure
1273 * we get an instruction fault as opposed to a data one
1275 madvise(p1
, PAGE_SIZE
, MADV_DONTNEED
);
1280 void test_executing_on_unreadable_memory(int *ptr
, u16 pkey
)
1287 p1
= get_pointer_to_instructions();
1288 lots_o_noops_around_write(&scratch
);
1289 ptr_contents
= read_ptr(p1
);
1290 dprintf2("ptr (%p) contents@%d: %x\n", p1
, __LINE__
, ptr_contents
);
1292 ret
= mprotect_pkey(p1
, PAGE_SIZE
, PROT_EXEC
, (u64
)pkey
);
1294 pkey_access_deny(pkey
);
1296 dprintf2("pkru: %x\n", rdpkru());
1299 * Make sure this is an *instruction* fault
1301 madvise(p1
, PAGE_SIZE
, MADV_DONTNEED
);
1302 lots_o_noops_around_write(&scratch
);
1303 do_not_expect_pk_fault("executing on PROT_EXEC memory");
1304 ptr_contents
= read_ptr(p1
);
1305 dprintf2("ptr (%p) contents@%d: %x\n", p1
, __LINE__
, ptr_contents
);
1306 expected_pk_fault(pkey
);
1309 void test_implicit_mprotect_exec_only_memory(int *ptr
, u16 pkey
)
1316 dprintf1("%s() start\n", __func__
);
1318 p1
= get_pointer_to_instructions();
1319 lots_o_noops_around_write(&scratch
);
1320 ptr_contents
= read_ptr(p1
);
1321 dprintf2("ptr (%p) contents@%d: %x\n", p1
, __LINE__
, ptr_contents
);
1323 /* Use a *normal* mprotect(), not mprotect_pkey(): */
1324 ret
= mprotect(p1
, PAGE_SIZE
, PROT_EXEC
);
1327 dprintf2("pkru: %x\n", rdpkru());
1329 /* Make sure this is an *instruction* fault */
1330 madvise(p1
, PAGE_SIZE
, MADV_DONTNEED
);
1331 lots_o_noops_around_write(&scratch
);
1332 do_not_expect_pk_fault("executing on PROT_EXEC memory");
1333 ptr_contents
= read_ptr(p1
);
1334 dprintf2("ptr (%p) contents@%d: %x\n", p1
, __LINE__
, ptr_contents
);
1335 expected_pk_fault(UNKNOWN_PKEY
);
1338 * Put the memory back to non-PROT_EXEC. Should clear the
1339 * exec-only pkey off the VMA and allow it to be readable
1340 * again. Go to PROT_NONE first to check for a kernel bug
1341 * that did not clear the pkey when doing PROT_NONE.
1343 ret
= mprotect(p1
, PAGE_SIZE
, PROT_NONE
);
1346 ret
= mprotect(p1
, PAGE_SIZE
, PROT_READ
|PROT_EXEC
);
1348 ptr_contents
= read_ptr(p1
);
1349 do_not_expect_pk_fault("plain read on recently PROT_EXEC area");
1352 void test_mprotect_pkey_on_unsupported_cpu(int *ptr
, u16 pkey
)
1354 int size
= PAGE_SIZE
;
1357 if (cpu_has_pku()) {
1358 dprintf1("SKIP: %s: no CPU support\n", __func__
);
1362 sret
= syscall(SYS_mprotect_key
, ptr
, size
, PROT_READ
, pkey
);
1363 pkey_assert(sret
< 0);
1366 void (*pkey_tests
[])(int *ptr
, u16 pkey
) = {
1367 test_read_of_write_disabled_region
,
1368 test_read_of_access_disabled_region
,
1369 test_write_of_write_disabled_region
,
1370 test_write_of_access_disabled_region
,
1371 test_kernel_write_of_access_disabled_region
,
1372 test_kernel_write_of_write_disabled_region
,
1373 test_kernel_gup_of_access_disabled_region
,
1374 test_kernel_gup_write_to_write_disabled_region
,
1375 test_executing_on_unreadable_memory
,
1376 test_implicit_mprotect_exec_only_memory
,
1377 test_ptrace_of_child
,
1378 test_pkey_syscalls_on_non_allocated_pkey
,
1379 test_pkey_syscalls_bad_args
,
1380 test_pkey_alloc_exhaust
,
1383 void run_tests_once(void)
1386 int prot
= PROT_READ
|PROT_WRITE
;
1388 for (test_nr
= 0; test_nr
< ARRAY_SIZE(pkey_tests
); test_nr
++) {
1390 int orig_pkru_faults
= pkru_faults
;
1392 dprintf1("======================\n");
1393 dprintf1("test %d preparing...\n", test_nr
);
1396 pkey
= alloc_random_pkey();
1397 dprintf1("test %d starting with pkey: %d\n", test_nr
, pkey
);
1398 ptr
= malloc_pkey(PAGE_SIZE
, prot
, pkey
);
1399 dprintf1("test %d starting...\n", test_nr
);
1400 pkey_tests
[test_nr
](ptr
, pkey
);
1401 dprintf1("freeing test memory: %p\n", ptr
);
1402 free_pkey_malloc(ptr
);
1403 sys_pkey_free(pkey
);
1405 dprintf1("pkru_faults: %d\n", pkru_faults
);
1406 dprintf1("orig_pkru_faults: %d\n", orig_pkru_faults
);
1411 printf("test %2d PASSED (iteration %d)\n", test_nr
, iteration_nr
);
1412 dprintf1("======================\n\n");
1417 void pkey_setup_shadow(void)
1419 shadow_pkru
= __rdpkru();
1424 int nr_iterations
= 22;
1428 printf("has pku: %d\n", cpu_has_pku());
1430 if (!cpu_has_pku()) {
1431 int size
= PAGE_SIZE
;
1434 printf("running PKEY tests for unsupported CPU/OS\n");
1436 ptr
= mmap(NULL
, size
, PROT_NONE
, MAP_ANONYMOUS
|MAP_PRIVATE
, -1, 0);
1437 assert(ptr
!= (void *)-1);
1438 test_mprotect_pkey_on_unsupported_cpu(ptr
, 1);
1442 pkey_setup_shadow();
1443 printf("startup pkru: %x\n", rdpkru());
1446 while (nr_iterations
-- > 0)
1449 printf("done (all tests OK)\n");