1 // SPDX-License-Identifier: GPL-2.0
3 * Tests x86 Memory Protection Keys (see Documentation/x86/protection-keys.txt)
5 * There are examples in here of:
6 * * how to set protection keys on memory
7 * * how to set/clear bits in PKRU (the rights register)
8 * * how to handle SEGV_PKRU signals and extract pkey-relevant
9 * information from the siginfo
12 * make sure KSM and KSM COW breaking works
13 * prefault pages in at malloc, or not
14 * protect MPX bounds tables with protection keys?
15 * make sure VMA splitting/merging is working correctly
16 * OOMs can destroy mm->mmap (see exit_mmap()), so make sure it is immune to pkeys
17 * look for pkey "leaks" where it is still set on a VMA but "freed" back to the kernel
18 * do a plain mprotect() to a mprotect_pkey() area and make sure the pkey sticks
21 * gcc -o protection_keys -O2 -g -std=gnu99 -pthread -Wall protection_keys.c -lrt -ldl -lm
22 * gcc -m32 -o protection_keys_32 -O2 -g -std=gnu99 -pthread -Wall protection_keys.c -lrt -ldl -lm
26 #include <linux/futex.h>
28 #include <sys/syscall.h>
38 #include <sys/types.h>
43 #include <sys/ptrace.h>
46 #include "pkey-helpers.h"
51 unsigned int shadow_pkru
;
53 #define HPAGE_SIZE (1UL<<21)
54 #define ARRAY_SIZE(x) (sizeof(x) / sizeof(*(x)))
55 #define ALIGN_UP(x, align_to) (((x) + ((align_to)-1)) & ~((align_to)-1))
56 #define ALIGN_DOWN(x, align_to) ((x) & ~((align_to)-1))
57 #define ALIGN_PTR_UP(p, ptr_align_to) ((typeof(p))ALIGN_UP((unsigned long)(p), ptr_align_to))
58 #define ALIGN_PTR_DOWN(p, ptr_align_to) ((typeof(p))ALIGN_DOWN((unsigned long)(p), ptr_align_to))
59 #define __stringify_1(x...) #x
60 #define __stringify(x...) __stringify_1(x)
62 #define PTR_ERR_ENOTSUP ((void *)-ENOTSUP)
65 char dprint_in_signal_buffer
[DPRINT_IN_SIGNAL_BUF_SIZE
];
67 extern void abort_hooks(void);
68 #define pkey_assert(condition) do { \
70 dprintf0("assert() at %s::%d test_nr: %d iteration: %d\n", \
72 test_nr, iteration_nr); \
73 dprintf0("errno at assert: %d", errno); \
78 #define raw_assert(cond) assert(cond)
80 void cat_into_file(char *str
, char *file
)
82 int fd
= open(file
, O_RDWR
);
85 dprintf2("%s(): writing '%s' to '%s'\n", __func__
, str
, file
);
87 * these need to be raw because they are called under
91 ret
= write(fd
, str
, strlen(str
));
92 if (ret
!= strlen(str
)) {
93 perror("write to file failed");
94 fprintf(stderr
, "filename: '%s' str: '%s'\n", file
, str
);
100 #if CONTROL_TRACING > 0
101 static int warned_tracing
;
102 int tracing_root_ok(void)
104 if (geteuid() != 0) {
106 fprintf(stderr
, "WARNING: not run as root, "
107 "can not do tracing control\n");
115 void tracing_on(void)
117 #if CONTROL_TRACING > 0
118 #define TRACEDIR "/sys/kernel/debug/tracing"
121 if (!tracing_root_ok())
124 sprintf(pidstr
, "%d", getpid());
125 cat_into_file("0", TRACEDIR
"/tracing_on");
126 cat_into_file("\n", TRACEDIR
"/trace");
128 cat_into_file("function_graph", TRACEDIR
"/current_tracer");
129 cat_into_file("1", TRACEDIR
"/options/funcgraph-proc");
131 cat_into_file("nop", TRACEDIR
"/current_tracer");
133 cat_into_file(pidstr
, TRACEDIR
"/set_ftrace_pid");
134 cat_into_file("1", TRACEDIR
"/tracing_on");
135 dprintf1("enabled tracing\n");
139 void tracing_off(void)
141 #if CONTROL_TRACING > 0
142 if (!tracing_root_ok())
144 cat_into_file("0", "/sys/kernel/debug/tracing/tracing_on");
148 void abort_hooks(void)
150 fprintf(stderr
, "running %s()...\n", __func__
);
152 #ifdef SLEEP_ON_ABORT
153 sleep(SLEEP_ON_ABORT
);
157 static inline void __page_o_noops(void)
159 /* 8-bytes of instruction * 512 bytes = 1 page */
160 asm(".rept 512 ; nopl 0x7eeeeeee(%eax) ; .endr");
164 * This attempts to have roughly a page of instructions followed by a few
165 * instructions that do a write, and another page of instructions. That
166 * way, we are pretty sure that the write is in the second page of
167 * instructions and has at least a page of padding behind it.
169 * *That* lets us be sure to madvise() away the write instruction, which
170 * will then fault, which makes sure that the fault code handles
171 * execute-only memory properly.
173 __attribute__((__aligned__(PAGE_SIZE
)))
174 void lots_o_noops_around_write(int *write_to_me
)
176 dprintf3("running %s()\n", __func__
);
178 /* Assume this happens in the second page of instructions: */
179 *write_to_me
= __LINE__
;
180 /* pad out by another page: */
182 dprintf3("%s() done\n", __func__
);
185 /* Define some kernel-like types */
192 #define SYS_mprotect_key 380
193 #define SYS_pkey_alloc 381
194 #define SYS_pkey_free 382
195 #define REG_IP_IDX REG_EIP
196 #define si_pkey_offset 0x14
198 #define SYS_mprotect_key 329
199 #define SYS_pkey_alloc 330
200 #define SYS_pkey_free 331
201 #define REG_IP_IDX REG_RIP
202 #define si_pkey_offset 0x20
205 void dump_mem(void *dumpme
, int len_bytes
)
207 char *c
= (void *)dumpme
;
210 for (i
= 0; i
< len_bytes
; i
+= sizeof(u64
)) {
211 u64
*ptr
= (u64
*)(c
+ i
);
212 dprintf1("dump[%03d][@%p]: %016jx\n", i
, ptr
, *ptr
);
216 #define SEGV_BNDERR 3 /* failed address bound checks */
217 #define SEGV_PKUERR 4
219 static char *si_code_str(int si_code
)
221 if (si_code
== SEGV_MAPERR
)
222 return "SEGV_MAPERR";
223 if (si_code
== SEGV_ACCERR
)
224 return "SEGV_ACCERR";
225 if (si_code
== SEGV_BNDERR
)
226 return "SEGV_BNDERR";
227 if (si_code
== SEGV_PKUERR
)
228 return "SEGV_PKUERR";
233 int last_si_pkey
= -1;
234 void signal_handler(int signum
, siginfo_t
*si
, void *vucontext
)
236 ucontext_t
*uctxt
= vucontext
;
246 dprint_in_signal
= 1;
247 dprintf1(">>>>===============SIGSEGV============================\n");
248 dprintf1("%s()::%d, pkru: 0x%x shadow: %x\n", __func__
, __LINE__
,
249 __rdpkru(), shadow_pkru
);
251 trapno
= uctxt
->uc_mcontext
.gregs
[REG_TRAPNO
];
252 ip
= uctxt
->uc_mcontext
.gregs
[REG_IP_IDX
];
253 fpregset
= uctxt
->uc_mcontext
.fpregs
;
254 fpregs
= (void *)fpregset
;
256 dprintf2("%s() trapno: %d ip: 0x%lx info->si_code: %s/%d\n", __func__
,
257 trapno
, ip
, si_code_str(si
->si_code
), si
->si_code
);
260 * 32-bit has some extra padding so that userspace can tell whether
261 * the XSTATE header is present in addition to the "legacy" FPU
262 * state. We just assume that it is here.
266 pkru_offset
= pkru_xstate_offset();
267 pkru_ptr
= (void *)(&fpregs
[pkru_offset
]);
269 dprintf1("siginfo: %p\n", si
);
270 dprintf1(" fpregs: %p\n", fpregs
);
272 * If we got a PKRU fault, we *HAVE* to have at least one bit set in
275 dprintf1("pkru_xstate_offset: %d\n", pkru_xstate_offset());
277 dump_mem(pkru_ptr
- 128, 256);
278 pkey_assert(*pkru_ptr
);
280 si_pkey_ptr
= (u32
*)(((u8
*)si
) + si_pkey_offset
);
281 dprintf1("si_pkey_ptr: %p\n", si_pkey_ptr
);
282 dump_mem(si_pkey_ptr
- 8, 24);
283 si_pkey
= *si_pkey_ptr
;
284 pkey_assert(si_pkey
< NR_PKEYS
);
285 last_si_pkey
= si_pkey
;
287 if ((si
->si_code
== SEGV_MAPERR
) ||
288 (si
->si_code
== SEGV_ACCERR
) ||
289 (si
->si_code
== SEGV_BNDERR
)) {
290 printf("non-PK si_code, exiting...\n");
294 dprintf1("signal pkru from xsave: %08x\n", *pkru_ptr
);
295 /* need __rdpkru() version so we do not do shadow_pkru checking */
296 dprintf1("signal pkru from pkru: %08x\n", __rdpkru());
297 dprintf1("si_pkey from siginfo: %jx\n", si_pkey
);
298 *(u64
*)pkru_ptr
= 0x00000000;
299 dprintf1("WARNING: set PRKU=0 to allow faulting instruction to continue\n");
301 dprintf1("<<<<==================================================\n");
305 "ERROR: In signal handler, page fault, trapno = %d, ip = %016lx\n",
307 fprintf(stderr
, "si_addr %p\n", si
->si_addr
);
308 fprintf(stderr
, "REG_ERR: %lx\n",
309 (unsigned long)uctxt
->uc_mcontext
.gregs
[REG_ERR
]);
312 fprintf(stderr
, "unexpected trap %d! at 0x%lx\n", trapno
, ip
);
313 fprintf(stderr
, "si_addr %p\n", si
->si_addr
);
314 fprintf(stderr
, "REG_ERR: %lx\n",
315 (unsigned long)uctxt
->uc_mcontext
.gregs
[REG_ERR
]);
318 dprint_in_signal
= 0;
321 int wait_all_children(void)
324 return waitpid(-1, &status
, 0);
329 dprint_in_signal
= 1;
330 dprintf2("[%d] SIGCHLD: %d\n", getpid(), x
);
331 dprint_in_signal
= 0;
334 void setup_sigsegv_handler(void)
337 struct sigaction newact
;
338 struct sigaction oldact
;
340 /* #PF is mapped to sigsegv */
341 int signum
= SIGSEGV
;
343 newact
.sa_handler
= 0;
344 newact
.sa_sigaction
= signal_handler
;
346 /*sigset_t - signals to block while in the handler */
347 /* get the old signal mask. */
348 rs
= sigprocmask(SIG_SETMASK
, 0, &newact
.sa_mask
);
349 pkey_assert(rs
== 0);
351 /* call sa_sigaction, not sa_handler*/
352 newact
.sa_flags
= SA_SIGINFO
;
354 newact
.sa_restorer
= 0; /* void(*)(), obsolete */
355 r
= sigaction(signum
, &newact
, &oldact
);
356 r
= sigaction(SIGALRM
, &newact
, &oldact
);
360 void setup_handlers(void)
362 signal(SIGCHLD
, &sig_chld
);
363 setup_sigsegv_handler();
366 pid_t
fork_lazy_child(void)
371 pkey_assert(forkret
>= 0);
372 dprintf3("[%d] fork() ret: %d\n", getpid(), forkret
);
377 dprintf1("child sleeping...\n");
384 void davecmp(void *_a
, void *_b
, int len
)
387 unsigned long *a
= _a
;
388 unsigned long *b
= _b
;
390 for (i
= 0; i
< len
/ sizeof(*a
); i
++) {
394 dprintf3("[%3d]: a: %016lx b: %016lx\n", i
, a
[i
], b
[i
]);
400 int fd
= open(f
, O_RDONLY
);
404 dprintf2("maps fd: %d\n", fd
);
406 nr_read
= read(fd
, &buf
[0], sizeof(buf
));
407 write(1, buf
, nr_read
);
408 } while (nr_read
> 0);
412 #define PKEY_DISABLE_ACCESS 0x1
413 #define PKEY_DISABLE_WRITE 0x2
415 u32
pkey_get(int pkey
, unsigned long flags
)
417 u32 mask
= (PKEY_DISABLE_ACCESS
|PKEY_DISABLE_WRITE
);
418 u32 pkru
= __rdpkru();
422 dprintf1("%s(pkey=%d, flags=%lx) = %x / %d\n",
423 __func__
, pkey
, flags
, 0, 0);
424 dprintf2("%s() raw pkru: %x\n", __func__
, pkru
);
426 shifted_pkru
= (pkru
>> (pkey
* PKRU_BITS_PER_PKEY
));
427 dprintf2("%s() shifted_pkru: %x\n", __func__
, shifted_pkru
);
428 masked_pkru
= shifted_pkru
& mask
;
429 dprintf2("%s() masked pkru: %x\n", __func__
, masked_pkru
);
431 * shift down the relevant bits to the lowest two, then
432 * mask off all the other high bits.
437 int pkey_set(int pkey
, unsigned long rights
, unsigned long flags
)
439 u32 mask
= (PKEY_DISABLE_ACCESS
|PKEY_DISABLE_WRITE
);
440 u32 old_pkru
= __rdpkru();
443 /* make sure that 'rights' only contains the bits we expect: */
444 assert(!(rights
& ~mask
));
448 /* mask out bits from pkey in old value: */
449 new_pkru
&= ~(mask
<< (pkey
* PKRU_BITS_PER_PKEY
));
450 /* OR in new bits for pkey: */
451 new_pkru
|= (rights
<< (pkey
* PKRU_BITS_PER_PKEY
));
455 dprintf3("%s(pkey=%d, rights=%lx, flags=%lx) = %x pkru now: %x old_pkru: %x\n",
456 __func__
, pkey
, rights
, flags
, 0, __rdpkru(), old_pkru
);
460 void pkey_disable_set(int pkey
, int flags
)
462 unsigned long syscall_flags
= 0;
465 u32 orig_pkru
= rdpkru();
467 dprintf1("START->%s(%d, 0x%x)\n", __func__
,
469 pkey_assert(flags
& (PKEY_DISABLE_ACCESS
| PKEY_DISABLE_WRITE
));
471 pkey_rights
= pkey_get(pkey
, syscall_flags
);
473 dprintf1("%s(%d) pkey_get(%d): %x\n", __func__
,
474 pkey
, pkey
, pkey_rights
);
475 pkey_assert(pkey_rights
>= 0);
477 pkey_rights
|= flags
;
479 ret
= pkey_set(pkey
, pkey_rights
, syscall_flags
);
481 /*pkru and flags have the same format */
482 shadow_pkru
|= flags
<< (pkey
* 2);
483 dprintf1("%s(%d) shadow: 0x%x\n", __func__
, pkey
, shadow_pkru
);
485 pkey_assert(ret
>= 0);
487 pkey_rights
= pkey_get(pkey
, syscall_flags
);
488 dprintf1("%s(%d) pkey_get(%d): %x\n", __func__
,
489 pkey
, pkey
, pkey_rights
);
491 dprintf1("%s(%d) pkru: 0x%x\n", __func__
, pkey
, rdpkru());
493 pkey_assert(rdpkru() > orig_pkru
);
494 dprintf1("END<---%s(%d, 0x%x)\n", __func__
,
498 void pkey_disable_clear(int pkey
, int flags
)
500 unsigned long syscall_flags
= 0;
502 int pkey_rights
= pkey_get(pkey
, syscall_flags
);
503 u32 orig_pkru
= rdpkru();
505 pkey_assert(flags
& (PKEY_DISABLE_ACCESS
| PKEY_DISABLE_WRITE
));
507 dprintf1("%s(%d) pkey_get(%d): %x\n", __func__
,
508 pkey
, pkey
, pkey_rights
);
509 pkey_assert(pkey_rights
>= 0);
511 pkey_rights
|= flags
;
513 ret
= pkey_set(pkey
, pkey_rights
, 0);
514 /* pkru and flags have the same format */
515 shadow_pkru
&= ~(flags
<< (pkey
* 2));
516 pkey_assert(ret
>= 0);
518 pkey_rights
= pkey_get(pkey
, syscall_flags
);
519 dprintf1("%s(%d) pkey_get(%d): %x\n", __func__
,
520 pkey
, pkey
, pkey_rights
);
522 dprintf1("%s(%d) pkru: 0x%x\n", __func__
, pkey
, rdpkru());
524 assert(rdpkru() > orig_pkru
);
527 void pkey_write_allow(int pkey
)
529 pkey_disable_clear(pkey
, PKEY_DISABLE_WRITE
);
531 void pkey_write_deny(int pkey
)
533 pkey_disable_set(pkey
, PKEY_DISABLE_WRITE
);
535 void pkey_access_allow(int pkey
)
537 pkey_disable_clear(pkey
, PKEY_DISABLE_ACCESS
);
539 void pkey_access_deny(int pkey
)
541 pkey_disable_set(pkey
, PKEY_DISABLE_ACCESS
);
544 int sys_mprotect_pkey(void *ptr
, size_t size
, unsigned long orig_prot
,
549 dprintf2("%s(0x%p, %zx, prot=%lx, pkey=%lx)\n", __func__
,
550 ptr
, size
, orig_prot
, pkey
);
553 sret
= syscall(SYS_mprotect_key
, ptr
, size
, orig_prot
, pkey
);
555 dprintf2("SYS_mprotect_key sret: %d\n", sret
);
556 dprintf2("SYS_mprotect_key prot: 0x%lx\n", orig_prot
);
557 dprintf2("SYS_mprotect_key failed, errno: %d\n", errno
);
558 if (DEBUG_LEVEL
>= 2)
559 perror("SYS_mprotect_pkey");
564 int sys_pkey_alloc(unsigned long flags
, unsigned long init_val
)
566 int ret
= syscall(SYS_pkey_alloc
, flags
, init_val
);
567 dprintf1("%s(flags=%lx, init_val=%lx) syscall ret: %d errno: %d\n",
568 __func__
, flags
, init_val
, ret
, errno
);
575 unsigned long init_val
= 0x0;
577 dprintf1("alloc_pkey()::%d, pkru: 0x%x shadow: %x\n",
578 __LINE__
, __rdpkru(), shadow_pkru
);
579 ret
= sys_pkey_alloc(0, init_val
);
581 * pkey_alloc() sets PKRU, so we need to reflect it in
584 dprintf4("alloc_pkey()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n",
585 __LINE__
, ret
, __rdpkru(), shadow_pkru
);
587 /* clear both the bits: */
588 shadow_pkru
&= ~(0x3 << (ret
* 2));
589 dprintf4("alloc_pkey()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n",
590 __LINE__
, ret
, __rdpkru(), shadow_pkru
);
592 * move the new state in from init_val
593 * (remember, we cheated and init_val == pkru format)
595 shadow_pkru
|= (init_val
<< (ret
* 2));
597 dprintf4("alloc_pkey()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n",
598 __LINE__
, ret
, __rdpkru(), shadow_pkru
);
599 dprintf1("alloc_pkey()::%d errno: %d\n", __LINE__
, errno
);
600 /* for shadow checking: */
602 dprintf4("alloc_pkey()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n",
603 __LINE__
, ret
, __rdpkru(), shadow_pkru
);
607 int sys_pkey_free(unsigned long pkey
)
609 int ret
= syscall(SYS_pkey_free
, pkey
);
610 dprintf1("%s(pkey=%ld) syscall ret: %d\n", __func__
, pkey
, ret
);
615 * I had a bug where pkey bits could be set by mprotect() but
616 * not cleared. This ensures we get lots of random bit sets
617 * and clears on the vma and pte pkey bits.
619 int alloc_random_pkey(void)
621 int max_nr_pkey_allocs
;
624 int alloced_pkeys
[NR_PKEYS
];
627 memset(alloced_pkeys
, 0, sizeof(alloced_pkeys
));
629 /* allocate every possible key and make a note of which ones we got */
630 max_nr_pkey_allocs
= NR_PKEYS
;
631 max_nr_pkey_allocs
= 1;
632 for (i
= 0; i
< max_nr_pkey_allocs
; i
++) {
633 int new_pkey
= alloc_pkey();
636 alloced_pkeys
[nr_alloced
++] = new_pkey
;
639 pkey_assert(nr_alloced
> 0);
640 /* select a random one out of the allocated ones */
641 random_index
= rand() % nr_alloced
;
642 ret
= alloced_pkeys
[random_index
];
643 /* now zero it out so we don't free it next */
644 alloced_pkeys
[random_index
] = 0;
646 /* go through the allocated ones that we did not want and free them */
647 for (i
= 0; i
< nr_alloced
; i
++) {
649 if (!alloced_pkeys
[i
])
651 free_ret
= sys_pkey_free(alloced_pkeys
[i
]);
652 pkey_assert(!free_ret
);
654 dprintf1("%s()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", __func__
,
655 __LINE__
, ret
, __rdpkru(), shadow_pkru
);
659 int mprotect_pkey(void *ptr
, size_t size
, unsigned long orig_prot
,
662 int nr_iterations
= random() % 100;
666 int rpkey
= alloc_random_pkey();
667 ret
= sys_mprotect_pkey(ptr
, size
, orig_prot
, pkey
);
668 dprintf1("sys_mprotect_pkey(%p, %zx, prot=0x%lx, pkey=%ld) ret: %d\n",
669 ptr
, size
, orig_prot
, pkey
, ret
);
670 if (nr_iterations
-- < 0)
673 dprintf1("%s()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", __func__
,
674 __LINE__
, ret
, __rdpkru(), shadow_pkru
);
675 sys_pkey_free(rpkey
);
676 dprintf1("%s()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", __func__
,
677 __LINE__
, ret
, __rdpkru(), shadow_pkru
);
679 pkey_assert(pkey
< NR_PKEYS
);
681 ret
= sys_mprotect_pkey(ptr
, size
, orig_prot
, pkey
);
682 dprintf1("mprotect_pkey(%p, %zx, prot=0x%lx, pkey=%ld) ret: %d\n",
683 ptr
, size
, orig_prot
, pkey
, ret
);
685 dprintf1("%s()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", __func__
,
686 __LINE__
, ret
, __rdpkru(), shadow_pkru
);
690 struct pkey_malloc_record
{
694 struct pkey_malloc_record
*pkey_malloc_records
;
695 long nr_pkey_malloc_records
;
696 void record_pkey_malloc(void *ptr
, long size
)
699 struct pkey_malloc_record
*rec
= NULL
;
701 for (i
= 0; i
< nr_pkey_malloc_records
; i
++) {
702 rec
= &pkey_malloc_records
[i
];
703 /* find a free record */
708 /* every record is full */
709 size_t old_nr_records
= nr_pkey_malloc_records
;
710 size_t new_nr_records
= (nr_pkey_malloc_records
* 2 + 1);
711 size_t new_size
= new_nr_records
* sizeof(struct pkey_malloc_record
);
712 dprintf2("new_nr_records: %zd\n", new_nr_records
);
713 dprintf2("new_size: %zd\n", new_size
);
714 pkey_malloc_records
= realloc(pkey_malloc_records
, new_size
);
715 pkey_assert(pkey_malloc_records
!= NULL
);
716 rec
= &pkey_malloc_records
[nr_pkey_malloc_records
];
718 * realloc() does not initialize memory, so zero it from
719 * the first new record all the way to the end.
721 for (i
= 0; i
< new_nr_records
- old_nr_records
; i
++)
722 memset(rec
+ i
, 0, sizeof(*rec
));
724 dprintf3("filling malloc record[%d/%p]: {%p, %ld}\n",
725 (int)(rec
- pkey_malloc_records
), rec
, ptr
, size
);
728 nr_pkey_malloc_records
++;
731 void free_pkey_malloc(void *ptr
)
735 dprintf3("%s(%p)\n", __func__
, ptr
);
736 for (i
= 0; i
< nr_pkey_malloc_records
; i
++) {
737 struct pkey_malloc_record
*rec
= &pkey_malloc_records
[i
];
738 dprintf4("looking for ptr %p at record[%ld/%p]: {%p, %ld}\n",
739 ptr
, i
, rec
, rec
->ptr
, rec
->size
);
740 if ((ptr
< rec
->ptr
) ||
741 (ptr
>= rec
->ptr
+ rec
->size
))
744 dprintf3("found ptr %p at record[%ld/%p]: {%p, %ld}\n",
745 ptr
, i
, rec
, rec
->ptr
, rec
->size
);
746 nr_pkey_malloc_records
--;
747 ret
= munmap(rec
->ptr
, rec
->size
);
748 dprintf3("munmap ret: %d\n", ret
);
750 dprintf3("clearing rec->ptr, rec: %p\n", rec
);
752 dprintf3("done clearing rec->ptr, rec: %p\n", rec
);
759 void *malloc_pkey_with_mprotect(long size
, int prot
, u16 pkey
)
765 dprintf1("doing %s(size=%ld, prot=0x%x, pkey=%d)\n", __func__
,
767 pkey_assert(pkey
< NR_PKEYS
);
768 ptr
= mmap(NULL
, size
, prot
, MAP_ANONYMOUS
|MAP_PRIVATE
, -1, 0);
769 pkey_assert(ptr
!= (void *)-1);
770 ret
= mprotect_pkey((void *)ptr
, PAGE_SIZE
, prot
, pkey
);
772 record_pkey_malloc(ptr
, size
);
775 dprintf1("%s() for pkey %d @ %p\n", __func__
, pkey
, ptr
);
779 void *malloc_pkey_anon_huge(long size
, int prot
, u16 pkey
)
784 dprintf1("doing %s(size=%ld, prot=0x%x, pkey=%d)\n", __func__
,
787 * Guarantee we can fit at least one huge page in the resulting
788 * allocation by allocating space for 2:
790 size
= ALIGN_UP(size
, HPAGE_SIZE
* 2);
791 ptr
= mmap(NULL
, size
, PROT_NONE
, MAP_ANONYMOUS
|MAP_PRIVATE
, -1, 0);
792 pkey_assert(ptr
!= (void *)-1);
793 record_pkey_malloc(ptr
, size
);
794 mprotect_pkey(ptr
, size
, prot
, pkey
);
796 dprintf1("unaligned ptr: %p\n", ptr
);
797 ptr
= ALIGN_PTR_UP(ptr
, HPAGE_SIZE
);
798 dprintf1(" aligned ptr: %p\n", ptr
);
799 ret
= madvise(ptr
, HPAGE_SIZE
, MADV_HUGEPAGE
);
800 dprintf1("MADV_HUGEPAGE ret: %d\n", ret
);
801 ret
= madvise(ptr
, HPAGE_SIZE
, MADV_WILLNEED
);
802 dprintf1("MADV_WILLNEED ret: %d\n", ret
);
803 memset(ptr
, 0, HPAGE_SIZE
);
805 dprintf1("mmap()'d thp for pkey %d @ %p\n", pkey
, ptr
);
809 int hugetlb_setup_ok
;
810 #define GET_NR_HUGE_PAGES 10
811 void setup_hugetlbfs(void)
817 if (geteuid() != 0) {
818 fprintf(stderr
, "WARNING: not run as root, can not do hugetlb test\n");
822 cat_into_file(__stringify(GET_NR_HUGE_PAGES
), "/proc/sys/vm/nr_hugepages");
825 * Now go make sure that we got the pages and that they
826 * are 2M pages. Someone might have made 1G the default.
828 fd
= open("/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages", O_RDONLY
);
830 perror("opening sysfs 2M hugetlb config");
834 /* -1 to guarantee leaving the trailing \0 */
835 err
= read(fd
, buf
, sizeof(buf
)-1);
838 perror("reading sysfs 2M hugetlb config");
842 if (atoi(buf
) != GET_NR_HUGE_PAGES
) {
843 fprintf(stderr
, "could not confirm 2M pages, got: '%s' expected %d\n",
844 buf
, GET_NR_HUGE_PAGES
);
848 hugetlb_setup_ok
= 1;
851 void *malloc_pkey_hugetlb(long size
, int prot
, u16 pkey
)
854 int flags
= MAP_ANONYMOUS
|MAP_PRIVATE
|MAP_HUGETLB
;
856 if (!hugetlb_setup_ok
)
857 return PTR_ERR_ENOTSUP
;
859 dprintf1("doing %s(%ld, %x, %x)\n", __func__
, size
, prot
, pkey
);
860 size
= ALIGN_UP(size
, HPAGE_SIZE
* 2);
861 pkey_assert(pkey
< NR_PKEYS
);
862 ptr
= mmap(NULL
, size
, PROT_NONE
, flags
, -1, 0);
863 pkey_assert(ptr
!= (void *)-1);
864 mprotect_pkey(ptr
, size
, prot
, pkey
);
866 record_pkey_malloc(ptr
, size
);
868 dprintf1("mmap()'d hugetlbfs for pkey %d @ %p\n", pkey
, ptr
);
872 void *malloc_pkey_mmap_dax(long size
, int prot
, u16 pkey
)
877 dprintf1("doing %s(size=%ld, prot=0x%x, pkey=%d)\n", __func__
,
879 pkey_assert(pkey
< NR_PKEYS
);
880 fd
= open("/dax/foo", O_RDWR
);
881 pkey_assert(fd
>= 0);
883 ptr
= mmap(0, size
, prot
, MAP_SHARED
, fd
, 0);
884 pkey_assert(ptr
!= (void *)-1);
886 mprotect_pkey(ptr
, size
, prot
, pkey
);
888 record_pkey_malloc(ptr
, size
);
890 dprintf1("mmap()'d for pkey %d @ %p\n", pkey
, ptr
);
895 void *(*pkey_malloc
[])(long size
, int prot
, u16 pkey
) = {
897 malloc_pkey_with_mprotect
,
898 malloc_pkey_anon_huge
,
900 /* can not do direct with the pkey_mprotect() API:
901 malloc_pkey_mmap_direct,
902 malloc_pkey_mmap_dax,
906 void *malloc_pkey(long size
, int prot
, u16 pkey
)
909 static int malloc_type
;
910 int nr_malloc_types
= ARRAY_SIZE(pkey_malloc
);
912 pkey_assert(pkey
< NR_PKEYS
);
915 pkey_assert(malloc_type
< nr_malloc_types
);
917 ret
= pkey_malloc
[malloc_type
](size
, prot
, pkey
);
918 pkey_assert(ret
!= (void *)-1);
921 if (malloc_type
>= nr_malloc_types
)
922 malloc_type
= (random()%nr_malloc_types
);
924 /* try again if the malloc_type we tried is unsupported */
925 if (ret
== PTR_ERR_ENOTSUP
)
931 dprintf3("%s(%ld, prot=%x, pkey=%x) returning: %p\n", __func__
,
932 size
, prot
, pkey
, ret
);
936 int last_pkru_faults
;
937 void expected_pk_fault(int pkey
)
939 dprintf2("%s(): last_pkru_faults: %d pkru_faults: %d\n",
940 __func__
, last_pkru_faults
, pkru_faults
);
941 dprintf2("%s(%d): last_si_pkey: %d\n", __func__
, pkey
, last_si_pkey
);
942 pkey_assert(last_pkru_faults
+ 1 == pkru_faults
);
943 pkey_assert(last_si_pkey
== pkey
);
945 * The signal handler shold have cleared out PKRU to let the
946 * test program continue. We now have to restore it.
951 __wrpkru(shadow_pkru
);
952 dprintf1("%s() set PKRU=%x to restore state after signal nuked it\n",
953 __func__
, shadow_pkru
);
954 last_pkru_faults
= pkru_faults
;
958 void do_not_expect_pk_fault(void)
960 pkey_assert(last_pkru_faults
== pkru_faults
);
963 int test_fds
[10] = { -1 };
965 void __save_test_fd(int fd
)
967 pkey_assert(fd
>= 0);
968 pkey_assert(nr_test_fds
< ARRAY_SIZE(test_fds
));
969 test_fds
[nr_test_fds
] = fd
;
973 int get_test_read_fd(void)
975 int test_fd
= open("/etc/passwd", O_RDONLY
);
976 __save_test_fd(test_fd
);
980 void close_test_fds(void)
984 for (i
= 0; i
< nr_test_fds
; i
++) {
993 #define barrier() __asm__ __volatile__("": : :"memory")
994 __attribute__((noinline
)) int read_ptr(int *ptr
)
997 * Keep GCC from optimizing this away somehow
1003 void test_read_of_write_disabled_region(int *ptr
, u16 pkey
)
1007 dprintf1("disabling write access to PKEY[1], doing read\n");
1008 pkey_write_deny(pkey
);
1009 ptr_contents
= read_ptr(ptr
);
1010 dprintf1("*ptr: %d\n", ptr_contents
);
1013 void test_read_of_access_disabled_region(int *ptr
, u16 pkey
)
1017 dprintf1("disabling access to PKEY[%02d], doing read @ %p\n", pkey
, ptr
);
1019 pkey_access_deny(pkey
);
1020 ptr_contents
= read_ptr(ptr
);
1021 dprintf1("*ptr: %d\n", ptr_contents
);
1022 expected_pk_fault(pkey
);
1024 void test_write_of_write_disabled_region(int *ptr
, u16 pkey
)
1026 dprintf1("disabling write access to PKEY[%02d], doing write\n", pkey
);
1027 pkey_write_deny(pkey
);
1029 expected_pk_fault(pkey
);
1031 void test_write_of_access_disabled_region(int *ptr
, u16 pkey
)
1033 dprintf1("disabling access to PKEY[%02d], doing write\n", pkey
);
1034 pkey_access_deny(pkey
);
1036 expected_pk_fault(pkey
);
1038 void test_kernel_write_of_access_disabled_region(int *ptr
, u16 pkey
)
1041 int test_fd
= get_test_read_fd();
1043 dprintf1("disabling access to PKEY[%02d], "
1044 "having kernel read() to buffer\n", pkey
);
1045 pkey_access_deny(pkey
);
1046 ret
= read(test_fd
, ptr
, 1);
1047 dprintf1("read ret: %d\n", ret
);
1050 void test_kernel_write_of_write_disabled_region(int *ptr
, u16 pkey
)
1053 int test_fd
= get_test_read_fd();
1055 pkey_write_deny(pkey
);
1056 ret
= read(test_fd
, ptr
, 100);
1057 dprintf1("read ret: %d\n", ret
);
1058 if (ret
< 0 && (DEBUG_LEVEL
> 0))
1059 perror("verbose read result (OK for this to be bad)");
1063 void test_kernel_gup_of_access_disabled_region(int *ptr
, u16 pkey
)
1065 int pipe_ret
, vmsplice_ret
;
1069 pipe_ret
= pipe(pipe_fds
);
1071 pkey_assert(pipe_ret
== 0);
1072 dprintf1("disabling access to PKEY[%02d], "
1073 "having kernel vmsplice from buffer\n", pkey
);
1074 pkey_access_deny(pkey
);
1076 iov
.iov_len
= PAGE_SIZE
;
1077 vmsplice_ret
= vmsplice(pipe_fds
[1], &iov
, 1, SPLICE_F_GIFT
);
1078 dprintf1("vmsplice() ret: %d\n", vmsplice_ret
);
1079 pkey_assert(vmsplice_ret
== -1);
1085 void test_kernel_gup_write_to_write_disabled_region(int *ptr
, u16 pkey
)
1087 int ignored
= 0xdada;
1089 int some_int
= __LINE__
;
1091 dprintf1("disabling write to PKEY[%02d], "
1092 "doing futex gunk in buffer\n", pkey
);
1094 pkey_write_deny(pkey
);
1095 futex_ret
= syscall(SYS_futex
, ptr
, FUTEX_WAIT
, some_int
-1, NULL
,
1097 if (DEBUG_LEVEL
> 0)
1099 dprintf1("futex() ret: %d\n", futex_ret
);
1102 /* Assumes that all pkeys other than 'pkey' are unallocated */
1103 void test_pkey_syscalls_on_non_allocated_pkey(int *ptr
, u16 pkey
)
1108 /* Note: 0 is the default pkey, so don't mess with it */
1109 for (i
= 1; i
< NR_PKEYS
; i
++) {
1113 dprintf1("trying get/set/free to non-allocated pkey: %2d\n", i
);
1114 err
= sys_pkey_free(i
);
1117 err
= sys_pkey_free(i
);
1120 err
= sys_mprotect_pkey(ptr
, PAGE_SIZE
, PROT_READ
, i
);
1125 /* Assumes that all pkeys other than 'pkey' are unallocated */
1126 void test_pkey_syscalls_bad_args(int *ptr
, u16 pkey
)
1129 int bad_pkey
= NR_PKEYS
+99;
1131 /* pass a known-invalid pkey in: */
1132 err
= sys_mprotect_pkey(ptr
, PAGE_SIZE
, PROT_READ
, bad_pkey
);
1136 /* Assumes that all pkeys other than 'pkey' are unallocated */
1137 void test_pkey_alloc_exhaust(int *ptr
, u16 pkey
)
1140 int allocated_pkeys
[NR_PKEYS
] = {0};
1141 int nr_allocated_pkeys
= 0;
1144 for (i
= 0; i
< NR_PKEYS
*2; i
++) {
1146 dprintf1("%s() alloc loop: %d\n", __func__
, i
);
1147 new_pkey
= alloc_pkey();
1148 dprintf4("%s()::%d, err: %d pkru: 0x%x shadow: 0x%x\n", __func__
,
1149 __LINE__
, err
, __rdpkru(), shadow_pkru
);
1150 rdpkru(); /* for shadow checking */
1151 dprintf2("%s() errno: %d ENOSPC: %d\n", __func__
, errno
, ENOSPC
);
1152 if ((new_pkey
== -1) && (errno
== ENOSPC
)) {
1153 dprintf2("%s() failed to allocate pkey after %d tries\n",
1154 __func__
, nr_allocated_pkeys
);
1157 pkey_assert(nr_allocated_pkeys
< NR_PKEYS
);
1158 allocated_pkeys
[nr_allocated_pkeys
++] = new_pkey
;
1161 dprintf3("%s()::%d\n", __func__
, __LINE__
);
1164 * ensure it did not reach the end of the loop without
1167 pkey_assert(i
< NR_PKEYS
*2);
1170 * There are 16 pkeys supported in hardware. One is taken
1171 * up for the default (0) and another can be taken up by
1172 * an execute-only mapping. Ensure that we can allocate
1173 * at least 14 (16-2).
1175 pkey_assert(i
>= NR_PKEYS
-2);
1177 for (i
= 0; i
< nr_allocated_pkeys
; i
++) {
1178 err
= sys_pkey_free(allocated_pkeys
[i
]);
1180 rdpkru(); /* for shadow checking */
1184 void test_ptrace_of_child(int *ptr
, u16 pkey
)
1186 __attribute__((__unused__
)) int peek_result
;
1192 * This is the "control" for our little expermient. Make sure
1193 * we can always access it when ptracing.
1195 int *plain_ptr_unaligned
= malloc(HPAGE_SIZE
);
1196 int *plain_ptr
= ALIGN_PTR_UP(plain_ptr_unaligned
, PAGE_SIZE
);
1199 * Fork a child which is an exact copy of this process, of course.
1200 * That means we can do all of our tests via ptrace() and then plain
1201 * memory access and ensure they work differently.
1203 child_pid
= fork_lazy_child();
1204 dprintf1("[%d] child pid: %d\n", getpid(), child_pid
);
1206 ret
= ptrace(PTRACE_ATTACH
, child_pid
, ignored
, ignored
);
1209 dprintf1("[%d] attach ret: %ld %d\n", getpid(), ret
, __LINE__
);
1210 pkey_assert(ret
!= -1);
1211 ret
= waitpid(child_pid
, &status
, WUNTRACED
);
1212 if ((ret
!= child_pid
) || !(WIFSTOPPED(status
))) {
1213 fprintf(stderr
, "weird waitpid result %ld stat %x\n",
1217 dprintf2("waitpid ret: %ld\n", ret
);
1218 dprintf2("waitpid status: %d\n", status
);
1220 pkey_access_deny(pkey
);
1221 pkey_write_deny(pkey
);
1223 /* Write access, untested for now:
1224 ret = ptrace(PTRACE_POKEDATA, child_pid, peek_at, data);
1225 pkey_assert(ret != -1);
1226 dprintf1("poke at %p: %ld\n", peek_at, ret);
1230 * Try to access the pkey-protected "ptr" via ptrace:
1232 ret
= ptrace(PTRACE_PEEKDATA
, child_pid
, ptr
, ignored
);
1233 /* expect it to work, without an error: */
1234 pkey_assert(ret
!= -1);
1235 /* Now access from the current task, and expect an exception: */
1236 peek_result
= read_ptr(ptr
);
1237 expected_pk_fault(pkey
);
1240 * Try to access the NON-pkey-protected "plain_ptr" via ptrace:
1242 ret
= ptrace(PTRACE_PEEKDATA
, child_pid
, plain_ptr
, ignored
);
1243 /* expect it to work, without an error: */
1244 pkey_assert(ret
!= -1);
1245 /* Now access from the current task, and expect NO exception: */
1246 peek_result
= read_ptr(plain_ptr
);
1247 do_not_expect_pk_fault();
1249 ret
= ptrace(PTRACE_DETACH
, child_pid
, ignored
, 0);
1250 pkey_assert(ret
!= -1);
1252 ret
= kill(child_pid
, SIGKILL
);
1253 pkey_assert(ret
!= -1);
1257 free(plain_ptr_unaligned
);
1260 void test_executing_on_unreadable_memory(int *ptr
, u16 pkey
)
1267 p1
= ALIGN_PTR_UP(&lots_o_noops_around_write
, PAGE_SIZE
);
1268 dprintf3("&lots_o_noops: %p\n", &lots_o_noops_around_write
);
1269 /* lots_o_noops_around_write should be page-aligned already */
1270 assert(p1
== &lots_o_noops_around_write
);
1272 /* Point 'p1' at the *second* page of the function: */
1275 madvise(p1
, PAGE_SIZE
, MADV_DONTNEED
);
1276 lots_o_noops_around_write(&scratch
);
1277 ptr_contents
= read_ptr(p1
);
1278 dprintf2("ptr (%p) contents@%d: %x\n", p1
, __LINE__
, ptr_contents
);
1280 ret
= mprotect_pkey(p1
, PAGE_SIZE
, PROT_EXEC
, (u64
)pkey
);
1282 pkey_access_deny(pkey
);
1284 dprintf2("pkru: %x\n", rdpkru());
1287 * Make sure this is an *instruction* fault
1289 madvise(p1
, PAGE_SIZE
, MADV_DONTNEED
);
1290 lots_o_noops_around_write(&scratch
);
1291 do_not_expect_pk_fault();
1292 ptr_contents
= read_ptr(p1
);
1293 dprintf2("ptr (%p) contents@%d: %x\n", p1
, __LINE__
, ptr_contents
);
1294 expected_pk_fault(pkey
);
1297 void test_mprotect_pkey_on_unsupported_cpu(int *ptr
, u16 pkey
)
1299 int size
= PAGE_SIZE
;
1302 if (cpu_has_pku()) {
1303 dprintf1("SKIP: %s: no CPU support\n", __func__
);
1307 sret
= syscall(SYS_mprotect_key
, ptr
, size
, PROT_READ
, pkey
);
1308 pkey_assert(sret
< 0);
1311 void (*pkey_tests
[])(int *ptr
, u16 pkey
) = {
1312 test_read_of_write_disabled_region
,
1313 test_read_of_access_disabled_region
,
1314 test_write_of_write_disabled_region
,
1315 test_write_of_access_disabled_region
,
1316 test_kernel_write_of_access_disabled_region
,
1317 test_kernel_write_of_write_disabled_region
,
1318 test_kernel_gup_of_access_disabled_region
,
1319 test_kernel_gup_write_to_write_disabled_region
,
1320 test_executing_on_unreadable_memory
,
1321 test_ptrace_of_child
,
1322 test_pkey_syscalls_on_non_allocated_pkey
,
1323 test_pkey_syscalls_bad_args
,
1324 test_pkey_alloc_exhaust
,
1327 void run_tests_once(void)
1330 int prot
= PROT_READ
|PROT_WRITE
;
1332 for (test_nr
= 0; test_nr
< ARRAY_SIZE(pkey_tests
); test_nr
++) {
1334 int orig_pkru_faults
= pkru_faults
;
1336 dprintf1("======================\n");
1337 dprintf1("test %d preparing...\n", test_nr
);
1340 pkey
= alloc_random_pkey();
1341 dprintf1("test %d starting with pkey: %d\n", test_nr
, pkey
);
1342 ptr
= malloc_pkey(PAGE_SIZE
, prot
, pkey
);
1343 dprintf1("test %d starting...\n", test_nr
);
1344 pkey_tests
[test_nr
](ptr
, pkey
);
1345 dprintf1("freeing test memory: %p\n", ptr
);
1346 free_pkey_malloc(ptr
);
1347 sys_pkey_free(pkey
);
1349 dprintf1("pkru_faults: %d\n", pkru_faults
);
1350 dprintf1("orig_pkru_faults: %d\n", orig_pkru_faults
);
1355 printf("test %2d PASSED (iteration %d)\n", test_nr
, iteration_nr
);
1356 dprintf1("======================\n\n");
1361 void pkey_setup_shadow(void)
1363 shadow_pkru
= __rdpkru();
1368 int nr_iterations
= 22;
1372 printf("has pku: %d\n", cpu_has_pku());
1374 if (!cpu_has_pku()) {
1375 int size
= PAGE_SIZE
;
1378 printf("running PKEY tests for unsupported CPU/OS\n");
1380 ptr
= mmap(NULL
, size
, PROT_NONE
, MAP_ANONYMOUS
|MAP_PRIVATE
, -1, 0);
1381 assert(ptr
!= (void *)-1);
1382 test_mprotect_pkey_on_unsupported_cpu(ptr
, 1);
1386 pkey_setup_shadow();
1387 printf("startup pkru: %x\n", rdpkru());
1390 while (nr_iterations
-- > 0)
1393 printf("done (all tests OK)\n");