]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
5f23f6d0 DH |
2 | /* |
3 | * Tests x86 Memory Protection Keys (see Documentation/x86/protection-keys.txt) | |
4 | * | |
5 | * There are examples in here of: | |
6 | * * how to set protection keys on memory | |
7 | * * how to set/clear bits in PKRU (the rights register) | |
8 | * * how to handle SEGV_PKRU signals and extract pkey-relevant | |
9 | * information from the siginfo | |
10 | * | |
11 | * Things to add: | |
12 | * make sure KSM and KSM COW breaking works | |
13 | * prefault pages in at malloc, or not | |
14 | * protect MPX bounds tables with protection keys? | |
15 | * make sure VMA splitting/merging is working correctly | |
16 | * OOMs can destroy mm->mmap (see exit_mmap()), so make sure it is immune to pkeys | |
17 | * look for pkey "leaks" where it is still set on a VMA but "freed" back to the kernel | |
18 | * do a plain mprotect() to a mprotect_pkey() area and make sure the pkey sticks | |
19 | * | |
20 | * Compile like this: | |
21 | * gcc -o protection_keys -O2 -g -std=gnu99 -pthread -Wall protection_keys.c -lrt -ldl -lm | |
22 | * gcc -m32 -o protection_keys_32 -O2 -g -std=gnu99 -pthread -Wall protection_keys.c -lrt -ldl -lm | |
23 | */ | |
24 | #define _GNU_SOURCE | |
25 | #include <errno.h> | |
26 | #include <linux/futex.h> | |
27 | #include <sys/time.h> | |
28 | #include <sys/syscall.h> | |
29 | #include <string.h> | |
30 | #include <stdio.h> | |
31 | #include <stdint.h> | |
32 | #include <stdbool.h> | |
33 | #include <signal.h> | |
34 | #include <assert.h> | |
35 | #include <stdlib.h> | |
36 | #include <ucontext.h> | |
37 | #include <sys/mman.h> | |
38 | #include <sys/types.h> | |
39 | #include <sys/wait.h> | |
40 | #include <sys/stat.h> | |
41 | #include <fcntl.h> | |
42 | #include <unistd.h> | |
43 | #include <sys/ptrace.h> | |
44 | #include <setjmp.h> | |
45 | ||
46 | #include "pkey-helpers.h" | |
47 | ||
48 | int iteration_nr = 1; | |
49 | int test_nr; | |
50 | ||
51 | unsigned int shadow_pkru; | |
52 | ||
53 | #define HPAGE_SIZE (1UL<<21) | |
54 | #define ARRAY_SIZE(x) (sizeof(x) / sizeof(*(x))) | |
55 | #define ALIGN_UP(x, align_to) (((x) + ((align_to)-1)) & ~((align_to)-1)) | |
56 | #define ALIGN_DOWN(x, align_to) ((x) & ~((align_to)-1)) | |
57 | #define ALIGN_PTR_UP(p, ptr_align_to) ((typeof(p))ALIGN_UP((unsigned long)(p), ptr_align_to)) | |
58 | #define ALIGN_PTR_DOWN(p, ptr_align_to) ((typeof(p))ALIGN_DOWN((unsigned long)(p), ptr_align_to)) | |
59 | #define __stringify_1(x...) #x | |
60 | #define __stringify(x...) __stringify_1(x) | |
61 | ||
62 | #define PTR_ERR_ENOTSUP ((void *)-ENOTSUP) | |
63 | ||
64 | int dprint_in_signal; | |
65 | char dprint_in_signal_buffer[DPRINT_IN_SIGNAL_BUF_SIZE]; | |
66 | ||
67 | extern void abort_hooks(void); | |
68 | #define pkey_assert(condition) do { \ | |
69 | if (!(condition)) { \ | |
70 | dprintf0("assert() at %s::%d test_nr: %d iteration: %d\n", \ | |
71 | __FILE__, __LINE__, \ | |
72 | test_nr, iteration_nr); \ | |
73 | dprintf0("errno at assert: %d", errno); \ | |
74 | abort_hooks(); \ | |
1686c433 | 75 | exit(__LINE__); \ |
5f23f6d0 DH |
76 | } \ |
77 | } while (0) | |
5f23f6d0 DH |
78 | |
79 | void cat_into_file(char *str, char *file) | |
80 | { | |
81 | int fd = open(file, O_RDWR); | |
82 | int ret; | |
83 | ||
84 | dprintf2("%s(): writing '%s' to '%s'\n", __func__, str, file); | |
85 | /* | |
86 | * these need to be raw because they are called under | |
87 | * pkey_assert() | |
88 | */ | |
1686c433 DH |
89 | if (fd < 0) { |
90 | fprintf(stderr, "error opening '%s'\n", str); | |
91 | perror("error: "); | |
92 | exit(__LINE__); | |
93 | } | |
94 | ||
5f23f6d0 DH |
95 | ret = write(fd, str, strlen(str)); |
96 | if (ret != strlen(str)) { | |
97 | perror("write to file failed"); | |
98 | fprintf(stderr, "filename: '%s' str: '%s'\n", file, str); | |
1686c433 | 99 | exit(__LINE__); |
5f23f6d0 DH |
100 | } |
101 | close(fd); | |
102 | } | |
103 | ||
104 | #if CONTROL_TRACING > 0 | |
105 | static int warned_tracing; | |
106 | int tracing_root_ok(void) | |
107 | { | |
108 | if (geteuid() != 0) { | |
109 | if (!warned_tracing) | |
110 | fprintf(stderr, "WARNING: not run as root, " | |
111 | "can not do tracing control\n"); | |
112 | warned_tracing = 1; | |
113 | return 0; | |
114 | } | |
115 | return 1; | |
116 | } | |
117 | #endif | |
118 | ||
119 | void tracing_on(void) | |
120 | { | |
121 | #if CONTROL_TRACING > 0 | |
122 | #define TRACEDIR "/sys/kernel/debug/tracing" | |
123 | char pidstr[32]; | |
124 | ||
125 | if (!tracing_root_ok()) | |
126 | return; | |
127 | ||
128 | sprintf(pidstr, "%d", getpid()); | |
129 | cat_into_file("0", TRACEDIR "/tracing_on"); | |
130 | cat_into_file("\n", TRACEDIR "/trace"); | |
131 | if (1) { | |
132 | cat_into_file("function_graph", TRACEDIR "/current_tracer"); | |
133 | cat_into_file("1", TRACEDIR "/options/funcgraph-proc"); | |
134 | } else { | |
135 | cat_into_file("nop", TRACEDIR "/current_tracer"); | |
136 | } | |
137 | cat_into_file(pidstr, TRACEDIR "/set_ftrace_pid"); | |
138 | cat_into_file("1", TRACEDIR "/tracing_on"); | |
139 | dprintf1("enabled tracing\n"); | |
140 | #endif | |
141 | } | |
142 | ||
143 | void tracing_off(void) | |
144 | { | |
145 | #if CONTROL_TRACING > 0 | |
146 | if (!tracing_root_ok()) | |
147 | return; | |
148 | cat_into_file("0", "/sys/kernel/debug/tracing/tracing_on"); | |
149 | #endif | |
150 | } | |
151 | ||
152 | void abort_hooks(void) | |
153 | { | |
154 | fprintf(stderr, "running %s()...\n", __func__); | |
155 | tracing_off(); | |
156 | #ifdef SLEEP_ON_ABORT | |
157 | sleep(SLEEP_ON_ABORT); | |
158 | #endif | |
159 | } | |
160 | ||
161 | static inline void __page_o_noops(void) | |
162 | { | |
163 | /* 8-bytes of instruction * 512 bytes = 1 page */ | |
164 | asm(".rept 512 ; nopl 0x7eeeeeee(%eax) ; .endr"); | |
165 | } | |
166 | ||
167 | /* | |
168 | * This attempts to have roughly a page of instructions followed by a few | |
169 | * instructions that do a write, and another page of instructions. That | |
170 | * way, we are pretty sure that the write is in the second page of | |
171 | * instructions and has at least a page of padding behind it. | |
172 | * | |
173 | * *That* lets us be sure to madvise() away the write instruction, which | |
174 | * will then fault, which makes sure that the fault code handles | |
175 | * execute-only memory properly. | |
176 | */ | |
177 | __attribute__((__aligned__(PAGE_SIZE))) | |
178 | void lots_o_noops_around_write(int *write_to_me) | |
179 | { | |
180 | dprintf3("running %s()\n", __func__); | |
181 | __page_o_noops(); | |
182 | /* Assume this happens in the second page of instructions: */ | |
183 | *write_to_me = __LINE__; | |
184 | /* pad out by another page: */ | |
185 | __page_o_noops(); | |
186 | dprintf3("%s() done\n", __func__); | |
187 | } | |
188 | ||
189 | /* Define some kernel-like types */ | |
190 | #define u8 uint8_t | |
191 | #define u16 uint16_t | |
192 | #define u32 uint32_t | |
193 | #define u64 uint64_t | |
194 | ||
195 | #ifdef __i386__ | |
693cb558 AL |
196 | |
197 | #ifndef SYS_mprotect_key | |
2981eb92 | 198 | # define SYS_mprotect_key 380 |
693cb558 | 199 | #endif |
2981eb92 | 200 | |
693cb558 | 201 | #ifndef SYS_pkey_alloc |
2981eb92 IM |
202 | # define SYS_pkey_alloc 381 |
203 | # define SYS_pkey_free 382 | |
693cb558 | 204 | #endif |
2981eb92 IM |
205 | |
206 | #define REG_IP_IDX REG_EIP | |
207 | #define si_pkey_offset 0x14 | |
693cb558 | 208 | |
5f23f6d0 | 209 | #else |
693cb558 AL |
210 | |
211 | #ifndef SYS_mprotect_key | |
2981eb92 | 212 | # define SYS_mprotect_key 329 |
693cb558 | 213 | #endif |
2981eb92 | 214 | |
693cb558 | 215 | #ifndef SYS_pkey_alloc |
2981eb92 IM |
216 | # define SYS_pkey_alloc 330 |
217 | # define SYS_pkey_free 331 | |
693cb558 | 218 | #endif |
2981eb92 IM |
219 | |
220 | #define REG_IP_IDX REG_RIP | |
221 | #define si_pkey_offset 0x20 | |
693cb558 | 222 | |
5f23f6d0 DH |
223 | #endif |
224 | ||
225 | void dump_mem(void *dumpme, int len_bytes) | |
226 | { | |
227 | char *c = (void *)dumpme; | |
228 | int i; | |
229 | ||
230 | for (i = 0; i < len_bytes; i += sizeof(u64)) { | |
231 | u64 *ptr = (u64 *)(c + i); | |
232 | dprintf1("dump[%03d][@%p]: %016jx\n", i, ptr, *ptr); | |
233 | } | |
234 | } | |
235 | ||
2981eb92 IM |
236 | /* Failed address bound checks: */ |
237 | #ifndef SEGV_BNDERR | |
238 | # define SEGV_BNDERR 3 | |
239 | #endif | |
240 | ||
241 | #ifndef SEGV_PKUERR | |
242 | # define SEGV_PKUERR 4 | |
243 | #endif | |
5f23f6d0 DH |
244 | |
245 | static char *si_code_str(int si_code) | |
246 | { | |
d12fe87e | 247 | if (si_code == SEGV_MAPERR) |
5f23f6d0 | 248 | return "SEGV_MAPERR"; |
d12fe87e | 249 | if (si_code == SEGV_ACCERR) |
5f23f6d0 | 250 | return "SEGV_ACCERR"; |
d12fe87e | 251 | if (si_code == SEGV_BNDERR) |
5f23f6d0 | 252 | return "SEGV_BNDERR"; |
d12fe87e | 253 | if (si_code == SEGV_PKUERR) |
5f23f6d0 DH |
254 | return "SEGV_PKUERR"; |
255 | return "UNKNOWN"; | |
256 | } | |
257 | ||
258 | int pkru_faults; | |
259 | int last_si_pkey = -1; | |
260 | void signal_handler(int signum, siginfo_t *si, void *vucontext) | |
261 | { | |
262 | ucontext_t *uctxt = vucontext; | |
263 | int trapno; | |
264 | unsigned long ip; | |
265 | char *fpregs; | |
266 | u32 *pkru_ptr; | |
91c49c2d | 267 | u64 siginfo_pkey; |
5f23f6d0 DH |
268 | u32 *si_pkey_ptr; |
269 | int pkru_offset; | |
270 | fpregset_t fpregset; | |
271 | ||
272 | dprint_in_signal = 1; | |
273 | dprintf1(">>>>===============SIGSEGV============================\n"); | |
274 | dprintf1("%s()::%d, pkru: 0x%x shadow: %x\n", __func__, __LINE__, | |
275 | __rdpkru(), shadow_pkru); | |
276 | ||
277 | trapno = uctxt->uc_mcontext.gregs[REG_TRAPNO]; | |
278 | ip = uctxt->uc_mcontext.gregs[REG_IP_IDX]; | |
279 | fpregset = uctxt->uc_mcontext.fpregs; | |
280 | fpregs = (void *)fpregset; | |
281 | ||
282 | dprintf2("%s() trapno: %d ip: 0x%lx info->si_code: %s/%d\n", __func__, | |
283 | trapno, ip, si_code_str(si->si_code), si->si_code); | |
284 | #ifdef __i386__ | |
285 | /* | |
286 | * 32-bit has some extra padding so that userspace can tell whether | |
287 | * the XSTATE header is present in addition to the "legacy" FPU | |
288 | * state. We just assume that it is here. | |
289 | */ | |
290 | fpregs += 0x70; | |
291 | #endif | |
292 | pkru_offset = pkru_xstate_offset(); | |
293 | pkru_ptr = (void *)(&fpregs[pkru_offset]); | |
294 | ||
295 | dprintf1("siginfo: %p\n", si); | |
296 | dprintf1(" fpregs: %p\n", fpregs); | |
297 | /* | |
298 | * If we got a PKRU fault, we *HAVE* to have at least one bit set in | |
299 | * here. | |
300 | */ | |
301 | dprintf1("pkru_xstate_offset: %d\n", pkru_xstate_offset()); | |
302 | if (DEBUG_LEVEL > 4) | |
303 | dump_mem(pkru_ptr - 128, 256); | |
304 | pkey_assert(*pkru_ptr); | |
305 | ||
5f23f6d0 DH |
306 | if ((si->si_code == SEGV_MAPERR) || |
307 | (si->si_code == SEGV_ACCERR) || | |
308 | (si->si_code == SEGV_BNDERR)) { | |
309 | printf("non-PK si_code, exiting...\n"); | |
310 | exit(4); | |
311 | } | |
312 | ||
1bf2c8a6 DH |
313 | si_pkey_ptr = (u32 *)(((u8 *)si) + si_pkey_offset); |
314 | dprintf1("si_pkey_ptr: %p\n", si_pkey_ptr); | |
315 | dump_mem((u8 *)si_pkey_ptr - 8, 24); | |
316 | siginfo_pkey = *si_pkey_ptr; | |
317 | pkey_assert(siginfo_pkey < NR_PKEYS); | |
318 | last_si_pkey = siginfo_pkey; | |
319 | ||
5f23f6d0 DH |
320 | dprintf1("signal pkru from xsave: %08x\n", *pkru_ptr); |
321 | /* need __rdpkru() version so we do not do shadow_pkru checking */ | |
322 | dprintf1("signal pkru from pkru: %08x\n", __rdpkru()); | |
91c49c2d | 323 | dprintf1("pkey from siginfo: %jx\n", siginfo_pkey); |
5f23f6d0 DH |
324 | *(u64 *)pkru_ptr = 0x00000000; |
325 | dprintf1("WARNING: set PRKU=0 to allow faulting instruction to continue\n"); | |
326 | pkru_faults++; | |
327 | dprintf1("<<<<==================================================\n"); | |
5f23f6d0 DH |
328 | dprint_in_signal = 0; |
329 | } | |
330 | ||
331 | int wait_all_children(void) | |
332 | { | |
333 | int status; | |
334 | return waitpid(-1, &status, 0); | |
335 | } | |
336 | ||
337 | void sig_chld(int x) | |
338 | { | |
339 | dprint_in_signal = 1; | |
340 | dprintf2("[%d] SIGCHLD: %d\n", getpid(), x); | |
341 | dprint_in_signal = 0; | |
342 | } | |
343 | ||
344 | void setup_sigsegv_handler(void) | |
345 | { | |
346 | int r, rs; | |
347 | struct sigaction newact; | |
348 | struct sigaction oldact; | |
349 | ||
350 | /* #PF is mapped to sigsegv */ | |
351 | int signum = SIGSEGV; | |
352 | ||
353 | newact.sa_handler = 0; | |
354 | newact.sa_sigaction = signal_handler; | |
355 | ||
356 | /*sigset_t - signals to block while in the handler */ | |
357 | /* get the old signal mask. */ | |
358 | rs = sigprocmask(SIG_SETMASK, 0, &newact.sa_mask); | |
359 | pkey_assert(rs == 0); | |
360 | ||
361 | /* call sa_sigaction, not sa_handler*/ | |
362 | newact.sa_flags = SA_SIGINFO; | |
363 | ||
364 | newact.sa_restorer = 0; /* void(*)(), obsolete */ | |
365 | r = sigaction(signum, &newact, &oldact); | |
366 | r = sigaction(SIGALRM, &newact, &oldact); | |
367 | pkey_assert(r == 0); | |
368 | } | |
369 | ||
370 | void setup_handlers(void) | |
371 | { | |
372 | signal(SIGCHLD, &sig_chld); | |
373 | setup_sigsegv_handler(); | |
374 | } | |
375 | ||
376 | pid_t fork_lazy_child(void) | |
377 | { | |
378 | pid_t forkret; | |
379 | ||
380 | forkret = fork(); | |
381 | pkey_assert(forkret >= 0); | |
382 | dprintf3("[%d] fork() ret: %d\n", getpid(), forkret); | |
383 | ||
384 | if (!forkret) { | |
385 | /* in the child */ | |
386 | while (1) { | |
387 | dprintf1("child sleeping...\n"); | |
388 | sleep(30); | |
389 | } | |
390 | } | |
391 | return forkret; | |
392 | } | |
393 | ||
2981eb92 IM |
394 | #ifndef PKEY_DISABLE_ACCESS |
395 | # define PKEY_DISABLE_ACCESS 0x1 | |
396 | #endif | |
397 | ||
398 | #ifndef PKEY_DISABLE_WRITE | |
399 | # define PKEY_DISABLE_WRITE 0x2 | |
400 | #endif | |
5f23f6d0 | 401 | |
2981eb92 | 402 | static u32 hw_pkey_get(int pkey, unsigned long flags) |
5f23f6d0 DH |
403 | { |
404 | u32 mask = (PKEY_DISABLE_ACCESS|PKEY_DISABLE_WRITE); | |
405 | u32 pkru = __rdpkru(); | |
406 | u32 shifted_pkru; | |
407 | u32 masked_pkru; | |
408 | ||
409 | dprintf1("%s(pkey=%d, flags=%lx) = %x / %d\n", | |
410 | __func__, pkey, flags, 0, 0); | |
411 | dprintf2("%s() raw pkru: %x\n", __func__, pkru); | |
412 | ||
413 | shifted_pkru = (pkru >> (pkey * PKRU_BITS_PER_PKEY)); | |
414 | dprintf2("%s() shifted_pkru: %x\n", __func__, shifted_pkru); | |
415 | masked_pkru = shifted_pkru & mask; | |
416 | dprintf2("%s() masked pkru: %x\n", __func__, masked_pkru); | |
417 | /* | |
418 | * shift down the relevant bits to the lowest two, then | |
419 | * mask off all the other high bits. | |
420 | */ | |
421 | return masked_pkru; | |
422 | } | |
423 | ||
2981eb92 | 424 | static int hw_pkey_set(int pkey, unsigned long rights, unsigned long flags) |
5f23f6d0 DH |
425 | { |
426 | u32 mask = (PKEY_DISABLE_ACCESS|PKEY_DISABLE_WRITE); | |
427 | u32 old_pkru = __rdpkru(); | |
428 | u32 new_pkru; | |
429 | ||
430 | /* make sure that 'rights' only contains the bits we expect: */ | |
431 | assert(!(rights & ~mask)); | |
432 | ||
433 | /* copy old pkru */ | |
434 | new_pkru = old_pkru; | |
435 | /* mask out bits from pkey in old value: */ | |
436 | new_pkru &= ~(mask << (pkey * PKRU_BITS_PER_PKEY)); | |
437 | /* OR in new bits for pkey: */ | |
438 | new_pkru |= (rights << (pkey * PKRU_BITS_PER_PKEY)); | |
439 | ||
440 | __wrpkru(new_pkru); | |
441 | ||
442 | dprintf3("%s(pkey=%d, rights=%lx, flags=%lx) = %x pkru now: %x old_pkru: %x\n", | |
443 | __func__, pkey, rights, flags, 0, __rdpkru(), old_pkru); | |
444 | return 0; | |
445 | } | |
446 | ||
447 | void pkey_disable_set(int pkey, int flags) | |
448 | { | |
449 | unsigned long syscall_flags = 0; | |
450 | int ret; | |
451 | int pkey_rights; | |
16846c2d | 452 | u32 orig_pkru = rdpkru(); |
5f23f6d0 DH |
453 | |
454 | dprintf1("START->%s(%d, 0x%x)\n", __func__, | |
455 | pkey, flags); | |
456 | pkey_assert(flags & (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE)); | |
457 | ||
2981eb92 | 458 | pkey_rights = hw_pkey_get(pkey, syscall_flags); |
5f23f6d0 | 459 | |
2981eb92 | 460 | dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__, |
5f23f6d0 DH |
461 | pkey, pkey, pkey_rights); |
462 | pkey_assert(pkey_rights >= 0); | |
463 | ||
464 | pkey_rights |= flags; | |
465 | ||
2981eb92 | 466 | ret = hw_pkey_set(pkey, pkey_rights, syscall_flags); |
5f23f6d0 DH |
467 | assert(!ret); |
468 | /*pkru and flags have the same format */ | |
469 | shadow_pkru |= flags << (pkey * 2); | |
470 | dprintf1("%s(%d) shadow: 0x%x\n", __func__, pkey, shadow_pkru); | |
471 | ||
472 | pkey_assert(ret >= 0); | |
473 | ||
2981eb92 IM |
474 | pkey_rights = hw_pkey_get(pkey, syscall_flags); |
475 | dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__, | |
5f23f6d0 DH |
476 | pkey, pkey, pkey_rights); |
477 | ||
478 | dprintf1("%s(%d) pkru: 0x%x\n", __func__, pkey, rdpkru()); | |
479 | if (flags) | |
480 | pkey_assert(rdpkru() > orig_pkru); | |
481 | dprintf1("END<---%s(%d, 0x%x)\n", __func__, | |
482 | pkey, flags); | |
483 | } | |
484 | ||
485 | void pkey_disable_clear(int pkey, int flags) | |
486 | { | |
487 | unsigned long syscall_flags = 0; | |
488 | int ret; | |
2981eb92 | 489 | int pkey_rights = hw_pkey_get(pkey, syscall_flags); |
5f23f6d0 DH |
490 | u32 orig_pkru = rdpkru(); |
491 | ||
492 | pkey_assert(flags & (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE)); | |
493 | ||
2981eb92 | 494 | dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__, |
5f23f6d0 DH |
495 | pkey, pkey, pkey_rights); |
496 | pkey_assert(pkey_rights >= 0); | |
497 | ||
498 | pkey_rights |= flags; | |
499 | ||
2981eb92 | 500 | ret = hw_pkey_set(pkey, pkey_rights, 0); |
5f23f6d0 DH |
501 | /* pkru and flags have the same format */ |
502 | shadow_pkru &= ~(flags << (pkey * 2)); | |
503 | pkey_assert(ret >= 0); | |
504 | ||
2981eb92 IM |
505 | pkey_rights = hw_pkey_get(pkey, syscall_flags); |
506 | dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__, | |
5f23f6d0 DH |
507 | pkey, pkey, pkey_rights); |
508 | ||
509 | dprintf1("%s(%d) pkru: 0x%x\n", __func__, pkey, rdpkru()); | |
510 | if (flags) | |
511 | assert(rdpkru() > orig_pkru); | |
512 | } | |
513 | ||
514 | void pkey_write_allow(int pkey) | |
515 | { | |
516 | pkey_disable_clear(pkey, PKEY_DISABLE_WRITE); | |
517 | } | |
518 | void pkey_write_deny(int pkey) | |
519 | { | |
520 | pkey_disable_set(pkey, PKEY_DISABLE_WRITE); | |
521 | } | |
522 | void pkey_access_allow(int pkey) | |
523 | { | |
524 | pkey_disable_clear(pkey, PKEY_DISABLE_ACCESS); | |
525 | } | |
526 | void pkey_access_deny(int pkey) | |
527 | { | |
528 | pkey_disable_set(pkey, PKEY_DISABLE_ACCESS); | |
529 | } | |
530 | ||
531 | int sys_mprotect_pkey(void *ptr, size_t size, unsigned long orig_prot, | |
532 | unsigned long pkey) | |
533 | { | |
534 | int sret; | |
535 | ||
536 | dprintf2("%s(0x%p, %zx, prot=%lx, pkey=%lx)\n", __func__, | |
537 | ptr, size, orig_prot, pkey); | |
538 | ||
539 | errno = 0; | |
540 | sret = syscall(SYS_mprotect_key, ptr, size, orig_prot, pkey); | |
541 | if (errno) { | |
542 | dprintf2("SYS_mprotect_key sret: %d\n", sret); | |
543 | dprintf2("SYS_mprotect_key prot: 0x%lx\n", orig_prot); | |
544 | dprintf2("SYS_mprotect_key failed, errno: %d\n", errno); | |
545 | if (DEBUG_LEVEL >= 2) | |
546 | perror("SYS_mprotect_pkey"); | |
547 | } | |
548 | return sret; | |
549 | } | |
550 | ||
551 | int sys_pkey_alloc(unsigned long flags, unsigned long init_val) | |
552 | { | |
553 | int ret = syscall(SYS_pkey_alloc, flags, init_val); | |
554 | dprintf1("%s(flags=%lx, init_val=%lx) syscall ret: %d errno: %d\n", | |
555 | __func__, flags, init_val, ret, errno); | |
556 | return ret; | |
557 | } | |
558 | ||
559 | int alloc_pkey(void) | |
560 | { | |
561 | int ret; | |
562 | unsigned long init_val = 0x0; | |
563 | ||
564 | dprintf1("alloc_pkey()::%d, pkru: 0x%x shadow: %x\n", | |
565 | __LINE__, __rdpkru(), shadow_pkru); | |
566 | ret = sys_pkey_alloc(0, init_val); | |
567 | /* | |
568 | * pkey_alloc() sets PKRU, so we need to reflect it in | |
569 | * shadow_pkru: | |
570 | */ | |
571 | dprintf4("alloc_pkey()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", | |
572 | __LINE__, ret, __rdpkru(), shadow_pkru); | |
573 | if (ret) { | |
574 | /* clear both the bits: */ | |
575 | shadow_pkru &= ~(0x3 << (ret * 2)); | |
576 | dprintf4("alloc_pkey()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", | |
577 | __LINE__, ret, __rdpkru(), shadow_pkru); | |
578 | /* | |
579 | * move the new state in from init_val | |
580 | * (remember, we cheated and init_val == pkru format) | |
581 | */ | |
582 | shadow_pkru |= (init_val << (ret * 2)); | |
583 | } | |
584 | dprintf4("alloc_pkey()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", | |
585 | __LINE__, ret, __rdpkru(), shadow_pkru); | |
586 | dprintf1("alloc_pkey()::%d errno: %d\n", __LINE__, errno); | |
587 | /* for shadow checking: */ | |
588 | rdpkru(); | |
589 | dprintf4("alloc_pkey()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", | |
590 | __LINE__, ret, __rdpkru(), shadow_pkru); | |
591 | return ret; | |
592 | } | |
593 | ||
594 | int sys_pkey_free(unsigned long pkey) | |
595 | { | |
596 | int ret = syscall(SYS_pkey_free, pkey); | |
597 | dprintf1("%s(pkey=%ld) syscall ret: %d\n", __func__, pkey, ret); | |
598 | return ret; | |
599 | } | |
600 | ||
601 | /* | |
602 | * I had a bug where pkey bits could be set by mprotect() but | |
603 | * not cleared. This ensures we get lots of random bit sets | |
604 | * and clears on the vma and pte pkey bits. | |
605 | */ | |
606 | int alloc_random_pkey(void) | |
607 | { | |
608 | int max_nr_pkey_allocs; | |
609 | int ret; | |
610 | int i; | |
611 | int alloced_pkeys[NR_PKEYS]; | |
612 | int nr_alloced = 0; | |
613 | int random_index; | |
614 | memset(alloced_pkeys, 0, sizeof(alloced_pkeys)); | |
615 | ||
616 | /* allocate every possible key and make a note of which ones we got */ | |
617 | max_nr_pkey_allocs = NR_PKEYS; | |
618 | max_nr_pkey_allocs = 1; | |
619 | for (i = 0; i < max_nr_pkey_allocs; i++) { | |
620 | int new_pkey = alloc_pkey(); | |
621 | if (new_pkey < 0) | |
622 | break; | |
623 | alloced_pkeys[nr_alloced++] = new_pkey; | |
624 | } | |
625 | ||
626 | pkey_assert(nr_alloced > 0); | |
627 | /* select a random one out of the allocated ones */ | |
628 | random_index = rand() % nr_alloced; | |
629 | ret = alloced_pkeys[random_index]; | |
630 | /* now zero it out so we don't free it next */ | |
631 | alloced_pkeys[random_index] = 0; | |
632 | ||
633 | /* go through the allocated ones that we did not want and free them */ | |
634 | for (i = 0; i < nr_alloced; i++) { | |
635 | int free_ret; | |
636 | if (!alloced_pkeys[i]) | |
637 | continue; | |
638 | free_ret = sys_pkey_free(alloced_pkeys[i]); | |
639 | pkey_assert(!free_ret); | |
640 | } | |
641 | dprintf1("%s()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", __func__, | |
642 | __LINE__, ret, __rdpkru(), shadow_pkru); | |
643 | return ret; | |
644 | } | |
645 | ||
646 | int mprotect_pkey(void *ptr, size_t size, unsigned long orig_prot, | |
647 | unsigned long pkey) | |
648 | { | |
649 | int nr_iterations = random() % 100; | |
650 | int ret; | |
651 | ||
652 | while (0) { | |
653 | int rpkey = alloc_random_pkey(); | |
654 | ret = sys_mprotect_pkey(ptr, size, orig_prot, pkey); | |
655 | dprintf1("sys_mprotect_pkey(%p, %zx, prot=0x%lx, pkey=%ld) ret: %d\n", | |
656 | ptr, size, orig_prot, pkey, ret); | |
657 | if (nr_iterations-- < 0) | |
658 | break; | |
659 | ||
660 | dprintf1("%s()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", __func__, | |
661 | __LINE__, ret, __rdpkru(), shadow_pkru); | |
662 | sys_pkey_free(rpkey); | |
663 | dprintf1("%s()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", __func__, | |
664 | __LINE__, ret, __rdpkru(), shadow_pkru); | |
665 | } | |
666 | pkey_assert(pkey < NR_PKEYS); | |
667 | ||
668 | ret = sys_mprotect_pkey(ptr, size, orig_prot, pkey); | |
669 | dprintf1("mprotect_pkey(%p, %zx, prot=0x%lx, pkey=%ld) ret: %d\n", | |
670 | ptr, size, orig_prot, pkey, ret); | |
671 | pkey_assert(!ret); | |
672 | dprintf1("%s()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", __func__, | |
673 | __LINE__, ret, __rdpkru(), shadow_pkru); | |
674 | return ret; | |
675 | } | |
676 | ||
677 | struct pkey_malloc_record { | |
678 | void *ptr; | |
679 | long size; | |
680 | }; | |
681 | struct pkey_malloc_record *pkey_malloc_records; | |
682 | long nr_pkey_malloc_records; | |
683 | void record_pkey_malloc(void *ptr, long size) | |
684 | { | |
685 | long i; | |
686 | struct pkey_malloc_record *rec = NULL; | |
687 | ||
688 | for (i = 0; i < nr_pkey_malloc_records; i++) { | |
689 | rec = &pkey_malloc_records[i]; | |
690 | /* find a free record */ | |
691 | if (rec) | |
692 | break; | |
693 | } | |
694 | if (!rec) { | |
695 | /* every record is full */ | |
696 | size_t old_nr_records = nr_pkey_malloc_records; | |
697 | size_t new_nr_records = (nr_pkey_malloc_records * 2 + 1); | |
698 | size_t new_size = new_nr_records * sizeof(struct pkey_malloc_record); | |
699 | dprintf2("new_nr_records: %zd\n", new_nr_records); | |
700 | dprintf2("new_size: %zd\n", new_size); | |
701 | pkey_malloc_records = realloc(pkey_malloc_records, new_size); | |
702 | pkey_assert(pkey_malloc_records != NULL); | |
703 | rec = &pkey_malloc_records[nr_pkey_malloc_records]; | |
704 | /* | |
705 | * realloc() does not initialize memory, so zero it from | |
706 | * the first new record all the way to the end. | |
707 | */ | |
708 | for (i = 0; i < new_nr_records - old_nr_records; i++) | |
709 | memset(rec + i, 0, sizeof(*rec)); | |
710 | } | |
711 | dprintf3("filling malloc record[%d/%p]: {%p, %ld}\n", | |
712 | (int)(rec - pkey_malloc_records), rec, ptr, size); | |
713 | rec->ptr = ptr; | |
714 | rec->size = size; | |
715 | nr_pkey_malloc_records++; | |
716 | } | |
717 | ||
718 | void free_pkey_malloc(void *ptr) | |
719 | { | |
720 | long i; | |
721 | int ret; | |
722 | dprintf3("%s(%p)\n", __func__, ptr); | |
723 | for (i = 0; i < nr_pkey_malloc_records; i++) { | |
724 | struct pkey_malloc_record *rec = &pkey_malloc_records[i]; | |
725 | dprintf4("looking for ptr %p at record[%ld/%p]: {%p, %ld}\n", | |
726 | ptr, i, rec, rec->ptr, rec->size); | |
727 | if ((ptr < rec->ptr) || | |
728 | (ptr >= rec->ptr + rec->size)) | |
729 | continue; | |
730 | ||
731 | dprintf3("found ptr %p at record[%ld/%p]: {%p, %ld}\n", | |
732 | ptr, i, rec, rec->ptr, rec->size); | |
733 | nr_pkey_malloc_records--; | |
734 | ret = munmap(rec->ptr, rec->size); | |
735 | dprintf3("munmap ret: %d\n", ret); | |
736 | pkey_assert(!ret); | |
737 | dprintf3("clearing rec->ptr, rec: %p\n", rec); | |
738 | rec->ptr = NULL; | |
739 | dprintf3("done clearing rec->ptr, rec: %p\n", rec); | |
740 | return; | |
741 | } | |
742 | pkey_assert(false); | |
743 | } | |
744 | ||
745 | ||
746 | void *malloc_pkey_with_mprotect(long size, int prot, u16 pkey) | |
747 | { | |
748 | void *ptr; | |
749 | int ret; | |
750 | ||
751 | rdpkru(); | |
752 | dprintf1("doing %s(size=%ld, prot=0x%x, pkey=%d)\n", __func__, | |
753 | size, prot, pkey); | |
754 | pkey_assert(pkey < NR_PKEYS); | |
755 | ptr = mmap(NULL, size, prot, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); | |
756 | pkey_assert(ptr != (void *)-1); | |
757 | ret = mprotect_pkey((void *)ptr, PAGE_SIZE, prot, pkey); | |
758 | pkey_assert(!ret); | |
759 | record_pkey_malloc(ptr, size); | |
760 | rdpkru(); | |
761 | ||
762 | dprintf1("%s() for pkey %d @ %p\n", __func__, pkey, ptr); | |
763 | return ptr; | |
764 | } | |
765 | ||
766 | void *malloc_pkey_anon_huge(long size, int prot, u16 pkey) | |
767 | { | |
768 | int ret; | |
769 | void *ptr; | |
770 | ||
771 | dprintf1("doing %s(size=%ld, prot=0x%x, pkey=%d)\n", __func__, | |
772 | size, prot, pkey); | |
773 | /* | |
774 | * Guarantee we can fit at least one huge page in the resulting | |
775 | * allocation by allocating space for 2: | |
776 | */ | |
777 | size = ALIGN_UP(size, HPAGE_SIZE * 2); | |
778 | ptr = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); | |
779 | pkey_assert(ptr != (void *)-1); | |
780 | record_pkey_malloc(ptr, size); | |
781 | mprotect_pkey(ptr, size, prot, pkey); | |
782 | ||
783 | dprintf1("unaligned ptr: %p\n", ptr); | |
784 | ptr = ALIGN_PTR_UP(ptr, HPAGE_SIZE); | |
785 | dprintf1(" aligned ptr: %p\n", ptr); | |
786 | ret = madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE); | |
787 | dprintf1("MADV_HUGEPAGE ret: %d\n", ret); | |
788 | ret = madvise(ptr, HPAGE_SIZE, MADV_WILLNEED); | |
789 | dprintf1("MADV_WILLNEED ret: %d\n", ret); | |
790 | memset(ptr, 0, HPAGE_SIZE); | |
791 | ||
792 | dprintf1("mmap()'d thp for pkey %d @ %p\n", pkey, ptr); | |
793 | return ptr; | |
794 | } | |
795 | ||
796 | int hugetlb_setup_ok; | |
797 | #define GET_NR_HUGE_PAGES 10 | |
798 | void setup_hugetlbfs(void) | |
799 | { | |
800 | int err; | |
801 | int fd; | |
5f23f6d0 DH |
802 | char buf[] = "123"; |
803 | ||
804 | if (geteuid() != 0) { | |
805 | fprintf(stderr, "WARNING: not run as root, can not do hugetlb test\n"); | |
806 | return; | |
807 | } | |
808 | ||
809 | cat_into_file(__stringify(GET_NR_HUGE_PAGES), "/proc/sys/vm/nr_hugepages"); | |
810 | ||
811 | /* | |
812 | * Now go make sure that we got the pages and that they | |
813 | * are 2M pages. Someone might have made 1G the default. | |
814 | */ | |
815 | fd = open("/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages", O_RDONLY); | |
816 | if (fd < 0) { | |
817 | perror("opening sysfs 2M hugetlb config"); | |
818 | return; | |
819 | } | |
820 | ||
821 | /* -1 to guarantee leaving the trailing \0 */ | |
822 | err = read(fd, buf, sizeof(buf)-1); | |
823 | close(fd); | |
824 | if (err <= 0) { | |
825 | perror("reading sysfs 2M hugetlb config"); | |
826 | return; | |
827 | } | |
828 | ||
829 | if (atoi(buf) != GET_NR_HUGE_PAGES) { | |
830 | fprintf(stderr, "could not confirm 2M pages, got: '%s' expected %d\n", | |
831 | buf, GET_NR_HUGE_PAGES); | |
832 | return; | |
833 | } | |
834 | ||
835 | hugetlb_setup_ok = 1; | |
836 | } | |
837 | ||
838 | void *malloc_pkey_hugetlb(long size, int prot, u16 pkey) | |
839 | { | |
840 | void *ptr; | |
841 | int flags = MAP_ANONYMOUS|MAP_PRIVATE|MAP_HUGETLB; | |
842 | ||
843 | if (!hugetlb_setup_ok) | |
844 | return PTR_ERR_ENOTSUP; | |
845 | ||
846 | dprintf1("doing %s(%ld, %x, %x)\n", __func__, size, prot, pkey); | |
847 | size = ALIGN_UP(size, HPAGE_SIZE * 2); | |
848 | pkey_assert(pkey < NR_PKEYS); | |
849 | ptr = mmap(NULL, size, PROT_NONE, flags, -1, 0); | |
850 | pkey_assert(ptr != (void *)-1); | |
851 | mprotect_pkey(ptr, size, prot, pkey); | |
852 | ||
853 | record_pkey_malloc(ptr, size); | |
854 | ||
855 | dprintf1("mmap()'d hugetlbfs for pkey %d @ %p\n", pkey, ptr); | |
856 | return ptr; | |
857 | } | |
858 | ||
859 | void *malloc_pkey_mmap_dax(long size, int prot, u16 pkey) | |
860 | { | |
861 | void *ptr; | |
862 | int fd; | |
863 | ||
864 | dprintf1("doing %s(size=%ld, prot=0x%x, pkey=%d)\n", __func__, | |
865 | size, prot, pkey); | |
866 | pkey_assert(pkey < NR_PKEYS); | |
867 | fd = open("/dax/foo", O_RDWR); | |
868 | pkey_assert(fd >= 0); | |
869 | ||
870 | ptr = mmap(0, size, prot, MAP_SHARED, fd, 0); | |
871 | pkey_assert(ptr != (void *)-1); | |
872 | ||
873 | mprotect_pkey(ptr, size, prot, pkey); | |
874 | ||
875 | record_pkey_malloc(ptr, size); | |
876 | ||
877 | dprintf1("mmap()'d for pkey %d @ %p\n", pkey, ptr); | |
878 | close(fd); | |
879 | return ptr; | |
880 | } | |
881 | ||
882 | void *(*pkey_malloc[])(long size, int prot, u16 pkey) = { | |
883 | ||
884 | malloc_pkey_with_mprotect, | |
885 | malloc_pkey_anon_huge, | |
886 | malloc_pkey_hugetlb | |
887 | /* can not do direct with the pkey_mprotect() API: | |
888 | malloc_pkey_mmap_direct, | |
889 | malloc_pkey_mmap_dax, | |
890 | */ | |
891 | }; | |
892 | ||
893 | void *malloc_pkey(long size, int prot, u16 pkey) | |
894 | { | |
895 | void *ret; | |
896 | static int malloc_type; | |
897 | int nr_malloc_types = ARRAY_SIZE(pkey_malloc); | |
898 | ||
899 | pkey_assert(pkey < NR_PKEYS); | |
900 | ||
901 | while (1) { | |
902 | pkey_assert(malloc_type < nr_malloc_types); | |
903 | ||
904 | ret = pkey_malloc[malloc_type](size, prot, pkey); | |
905 | pkey_assert(ret != (void *)-1); | |
906 | ||
907 | malloc_type++; | |
908 | if (malloc_type >= nr_malloc_types) | |
909 | malloc_type = (random()%nr_malloc_types); | |
910 | ||
911 | /* try again if the malloc_type we tried is unsupported */ | |
912 | if (ret == PTR_ERR_ENOTSUP) | |
913 | continue; | |
914 | ||
915 | break; | |
916 | } | |
917 | ||
918 | dprintf3("%s(%ld, prot=%x, pkey=%x) returning: %p\n", __func__, | |
919 | size, prot, pkey, ret); | |
920 | return ret; | |
921 | } | |
922 | ||
923 | int last_pkru_faults; | |
4352a046 | 924 | #define UNKNOWN_PKEY -2 |
5f23f6d0 DH |
925 | void expected_pk_fault(int pkey) |
926 | { | |
927 | dprintf2("%s(): last_pkru_faults: %d pkru_faults: %d\n", | |
928 | __func__, last_pkru_faults, pkru_faults); | |
929 | dprintf2("%s(%d): last_si_pkey: %d\n", __func__, pkey, last_si_pkey); | |
930 | pkey_assert(last_pkru_faults + 1 == pkru_faults); | |
4352a046 DH |
931 | |
932 | /* | |
933 | * For exec-only memory, we do not know the pkey in | |
934 | * advance, so skip this check. | |
935 | */ | |
936 | if (pkey != UNKNOWN_PKEY) | |
937 | pkey_assert(last_si_pkey == pkey); | |
938 | ||
5f23f6d0 DH |
939 | /* |
940 | * The signal handler shold have cleared out PKRU to let the | |
941 | * test program continue. We now have to restore it. | |
942 | */ | |
943 | if (__rdpkru() != 0) | |
944 | pkey_assert(0); | |
945 | ||
946 | __wrpkru(shadow_pkru); | |
947 | dprintf1("%s() set PKRU=%x to restore state after signal nuked it\n", | |
948 | __func__, shadow_pkru); | |
949 | last_pkru_faults = pkru_faults; | |
950 | last_si_pkey = -1; | |
951 | } | |
952 | ||
99478798 DH |
953 | #define do_not_expect_pk_fault(msg) do { \ |
954 | if (last_pkru_faults != pkru_faults) \ | |
955 | dprintf0("unexpected PK fault: %s\n", msg); \ | |
956 | pkey_assert(last_pkru_faults == pkru_faults); \ | |
957 | } while (0) | |
5f23f6d0 DH |
958 | |
959 | int test_fds[10] = { -1 }; | |
960 | int nr_test_fds; | |
961 | void __save_test_fd(int fd) | |
962 | { | |
963 | pkey_assert(fd >= 0); | |
964 | pkey_assert(nr_test_fds < ARRAY_SIZE(test_fds)); | |
965 | test_fds[nr_test_fds] = fd; | |
966 | nr_test_fds++; | |
967 | } | |
968 | ||
969 | int get_test_read_fd(void) | |
970 | { | |
971 | int test_fd = open("/etc/passwd", O_RDONLY); | |
972 | __save_test_fd(test_fd); | |
973 | return test_fd; | |
974 | } | |
975 | ||
976 | void close_test_fds(void) | |
977 | { | |
978 | int i; | |
979 | ||
980 | for (i = 0; i < nr_test_fds; i++) { | |
981 | if (test_fds[i] < 0) | |
982 | continue; | |
983 | close(test_fds[i]); | |
984 | test_fds[i] = -1; | |
985 | } | |
986 | nr_test_fds = 0; | |
987 | } | |
988 | ||
989 | #define barrier() __asm__ __volatile__("": : :"memory") | |
990 | __attribute__((noinline)) int read_ptr(int *ptr) | |
991 | { | |
992 | /* | |
993 | * Keep GCC from optimizing this away somehow | |
994 | */ | |
995 | barrier(); | |
996 | return *ptr; | |
997 | } | |
998 | ||
999 | void test_read_of_write_disabled_region(int *ptr, u16 pkey) | |
1000 | { | |
1001 | int ptr_contents; | |
1002 | ||
1003 | dprintf1("disabling write access to PKEY[1], doing read\n"); | |
1004 | pkey_write_deny(pkey); | |
1005 | ptr_contents = read_ptr(ptr); | |
1006 | dprintf1("*ptr: %d\n", ptr_contents); | |
1007 | dprintf1("\n"); | |
1008 | } | |
1009 | void test_read_of_access_disabled_region(int *ptr, u16 pkey) | |
1010 | { | |
1011 | int ptr_contents; | |
1012 | ||
1013 | dprintf1("disabling access to PKEY[%02d], doing read @ %p\n", pkey, ptr); | |
1014 | rdpkru(); | |
1015 | pkey_access_deny(pkey); | |
1016 | ptr_contents = read_ptr(ptr); | |
1017 | dprintf1("*ptr: %d\n", ptr_contents); | |
1018 | expected_pk_fault(pkey); | |
1019 | } | |
1020 | void test_write_of_write_disabled_region(int *ptr, u16 pkey) | |
1021 | { | |
1022 | dprintf1("disabling write access to PKEY[%02d], doing write\n", pkey); | |
1023 | pkey_write_deny(pkey); | |
1024 | *ptr = __LINE__; | |
1025 | expected_pk_fault(pkey); | |
1026 | } | |
1027 | void test_write_of_access_disabled_region(int *ptr, u16 pkey) | |
1028 | { | |
1029 | dprintf1("disabling access to PKEY[%02d], doing write\n", pkey); | |
1030 | pkey_access_deny(pkey); | |
1031 | *ptr = __LINE__; | |
1032 | expected_pk_fault(pkey); | |
1033 | } | |
1034 | void test_kernel_write_of_access_disabled_region(int *ptr, u16 pkey) | |
1035 | { | |
1036 | int ret; | |
1037 | int test_fd = get_test_read_fd(); | |
1038 | ||
1039 | dprintf1("disabling access to PKEY[%02d], " | |
1040 | "having kernel read() to buffer\n", pkey); | |
1041 | pkey_access_deny(pkey); | |
1042 | ret = read(test_fd, ptr, 1); | |
1043 | dprintf1("read ret: %d\n", ret); | |
1044 | pkey_assert(ret); | |
1045 | } | |
1046 | void test_kernel_write_of_write_disabled_region(int *ptr, u16 pkey) | |
1047 | { | |
1048 | int ret; | |
1049 | int test_fd = get_test_read_fd(); | |
1050 | ||
1051 | pkey_write_deny(pkey); | |
1052 | ret = read(test_fd, ptr, 100); | |
1053 | dprintf1("read ret: %d\n", ret); | |
1054 | if (ret < 0 && (DEBUG_LEVEL > 0)) | |
1055 | perror("verbose read result (OK for this to be bad)"); | |
1056 | pkey_assert(ret); | |
1057 | } | |
1058 | ||
1059 | void test_kernel_gup_of_access_disabled_region(int *ptr, u16 pkey) | |
1060 | { | |
1061 | int pipe_ret, vmsplice_ret; | |
1062 | struct iovec iov; | |
1063 | int pipe_fds[2]; | |
1064 | ||
1065 | pipe_ret = pipe(pipe_fds); | |
1066 | ||
1067 | pkey_assert(pipe_ret == 0); | |
1068 | dprintf1("disabling access to PKEY[%02d], " | |
1069 | "having kernel vmsplice from buffer\n", pkey); | |
1070 | pkey_access_deny(pkey); | |
1071 | iov.iov_base = ptr; | |
1072 | iov.iov_len = PAGE_SIZE; | |
1073 | vmsplice_ret = vmsplice(pipe_fds[1], &iov, 1, SPLICE_F_GIFT); | |
1074 | dprintf1("vmsplice() ret: %d\n", vmsplice_ret); | |
1075 | pkey_assert(vmsplice_ret == -1); | |
1076 | ||
1077 | close(pipe_fds[0]); | |
1078 | close(pipe_fds[1]); | |
1079 | } | |
1080 | ||
1081 | void test_kernel_gup_write_to_write_disabled_region(int *ptr, u16 pkey) | |
1082 | { | |
1083 | int ignored = 0xdada; | |
1084 | int futex_ret; | |
1085 | int some_int = __LINE__; | |
1086 | ||
1087 | dprintf1("disabling write to PKEY[%02d], " | |
1088 | "doing futex gunk in buffer\n", pkey); | |
1089 | *ptr = some_int; | |
1090 | pkey_write_deny(pkey); | |
1091 | futex_ret = syscall(SYS_futex, ptr, FUTEX_WAIT, some_int-1, NULL, | |
1092 | &ignored, ignored); | |
1093 | if (DEBUG_LEVEL > 0) | |
1094 | perror("futex"); | |
1095 | dprintf1("futex() ret: %d\n", futex_ret); | |
1096 | } | |
1097 | ||
1098 | /* Assumes that all pkeys other than 'pkey' are unallocated */ | |
1099 | void test_pkey_syscalls_on_non_allocated_pkey(int *ptr, u16 pkey) | |
1100 | { | |
1101 | int err; | |
1102 | int i; | |
1103 | ||
1104 | /* Note: 0 is the default pkey, so don't mess with it */ | |
1105 | for (i = 1; i < NR_PKEYS; i++) { | |
1106 | if (pkey == i) | |
1107 | continue; | |
1108 | ||
1109 | dprintf1("trying get/set/free to non-allocated pkey: %2d\n", i); | |
1110 | err = sys_pkey_free(i); | |
1111 | pkey_assert(err); | |
1112 | ||
5f23f6d0 DH |
1113 | err = sys_pkey_free(i); |
1114 | pkey_assert(err); | |
1115 | ||
1116 | err = sys_mprotect_pkey(ptr, PAGE_SIZE, PROT_READ, i); | |
1117 | pkey_assert(err); | |
1118 | } | |
1119 | } | |
1120 | ||
1121 | /* Assumes that all pkeys other than 'pkey' are unallocated */ | |
1122 | void test_pkey_syscalls_bad_args(int *ptr, u16 pkey) | |
1123 | { | |
1124 | int err; | |
5f23f6d0 DH |
1125 | int bad_pkey = NR_PKEYS+99; |
1126 | ||
5f23f6d0 DH |
1127 | /* pass a known-invalid pkey in: */ |
1128 | err = sys_mprotect_pkey(ptr, PAGE_SIZE, PROT_READ, bad_pkey); | |
1129 | pkey_assert(err); | |
1130 | } | |
1131 | ||
1132 | /* Assumes that all pkeys other than 'pkey' are unallocated */ | |
1133 | void test_pkey_alloc_exhaust(int *ptr, u16 pkey) | |
1134 | { | |
5f23f6d0 DH |
1135 | int err; |
1136 | int allocated_pkeys[NR_PKEYS] = {0}; | |
1137 | int nr_allocated_pkeys = 0; | |
1138 | int i; | |
1139 | ||
1140 | for (i = 0; i < NR_PKEYS*2; i++) { | |
1141 | int new_pkey; | |
1142 | dprintf1("%s() alloc loop: %d\n", __func__, i); | |
1143 | new_pkey = alloc_pkey(); | |
1144 | dprintf4("%s()::%d, err: %d pkru: 0x%x shadow: 0x%x\n", __func__, | |
1145 | __LINE__, err, __rdpkru(), shadow_pkru); | |
1146 | rdpkru(); /* for shadow checking */ | |
1147 | dprintf2("%s() errno: %d ENOSPC: %d\n", __func__, errno, ENOSPC); | |
1148 | if ((new_pkey == -1) && (errno == ENOSPC)) { | |
1149 | dprintf2("%s() failed to allocate pkey after %d tries\n", | |
1150 | __func__, nr_allocated_pkeys); | |
1151 | break; | |
1152 | } | |
1153 | pkey_assert(nr_allocated_pkeys < NR_PKEYS); | |
1154 | allocated_pkeys[nr_allocated_pkeys++] = new_pkey; | |
1155 | } | |
1156 | ||
1157 | dprintf3("%s()::%d\n", __func__, __LINE__); | |
1158 | ||
1159 | /* | |
1160 | * ensure it did not reach the end of the loop without | |
1161 | * failure: | |
1162 | */ | |
1163 | pkey_assert(i < NR_PKEYS*2); | |
1164 | ||
1165 | /* | |
ae378045 DH |
1166 | * There are 16 pkeys supported in hardware. Three are |
1167 | * allocated by the time we get here: | |
1168 | * 1. The default key (0) | |
1169 | * 2. One possibly consumed by an execute-only mapping. | |
1170 | * 3. One allocated by the test code and passed in via | |
1171 | * 'pkey' to this function. | |
1172 | * Ensure that we can allocate at least another 13 (16-3). | |
5f23f6d0 | 1173 | */ |
ae378045 | 1174 | pkey_assert(i >= NR_PKEYS-3); |
5f23f6d0 DH |
1175 | |
1176 | for (i = 0; i < nr_allocated_pkeys; i++) { | |
1177 | err = sys_pkey_free(allocated_pkeys[i]); | |
1178 | pkey_assert(!err); | |
1179 | rdpkru(); /* for shadow checking */ | |
1180 | } | |
1181 | } | |
1182 | ||
1183 | void test_ptrace_of_child(int *ptr, u16 pkey) | |
1184 | { | |
1185 | __attribute__((__unused__)) int peek_result; | |
1186 | pid_t child_pid; | |
1187 | void *ignored = 0; | |
1188 | long ret; | |
1189 | int status; | |
1190 | /* | |
1191 | * This is the "control" for our little expermient. Make sure | |
1192 | * we can always access it when ptracing. | |
1193 | */ | |
1194 | int *plain_ptr_unaligned = malloc(HPAGE_SIZE); | |
1195 | int *plain_ptr = ALIGN_PTR_UP(plain_ptr_unaligned, PAGE_SIZE); | |
1196 | ||
1197 | /* | |
1198 | * Fork a child which is an exact copy of this process, of course. | |
1199 | * That means we can do all of our tests via ptrace() and then plain | |
1200 | * memory access and ensure they work differently. | |
1201 | */ | |
1202 | child_pid = fork_lazy_child(); | |
1203 | dprintf1("[%d] child pid: %d\n", getpid(), child_pid); | |
1204 | ||
1205 | ret = ptrace(PTRACE_ATTACH, child_pid, ignored, ignored); | |
1206 | if (ret) | |
1207 | perror("attach"); | |
1208 | dprintf1("[%d] attach ret: %ld %d\n", getpid(), ret, __LINE__); | |
1209 | pkey_assert(ret != -1); | |
1210 | ret = waitpid(child_pid, &status, WUNTRACED); | |
1211 | if ((ret != child_pid) || !(WIFSTOPPED(status))) { | |
1212 | fprintf(stderr, "weird waitpid result %ld stat %x\n", | |
1213 | ret, status); | |
1214 | pkey_assert(0); | |
1215 | } | |
1216 | dprintf2("waitpid ret: %ld\n", ret); | |
1217 | dprintf2("waitpid status: %d\n", status); | |
1218 | ||
1219 | pkey_access_deny(pkey); | |
1220 | pkey_write_deny(pkey); | |
1221 | ||
1222 | /* Write access, untested for now: | |
1223 | ret = ptrace(PTRACE_POKEDATA, child_pid, peek_at, data); | |
1224 | pkey_assert(ret != -1); | |
1225 | dprintf1("poke at %p: %ld\n", peek_at, ret); | |
1226 | */ | |
1227 | ||
1228 | /* | |
1229 | * Try to access the pkey-protected "ptr" via ptrace: | |
1230 | */ | |
1231 | ret = ptrace(PTRACE_PEEKDATA, child_pid, ptr, ignored); | |
1232 | /* expect it to work, without an error: */ | |
1233 | pkey_assert(ret != -1); | |
1234 | /* Now access from the current task, and expect an exception: */ | |
1235 | peek_result = read_ptr(ptr); | |
1236 | expected_pk_fault(pkey); | |
1237 | ||
1238 | /* | |
1239 | * Try to access the NON-pkey-protected "plain_ptr" via ptrace: | |
1240 | */ | |
1241 | ret = ptrace(PTRACE_PEEKDATA, child_pid, plain_ptr, ignored); | |
1242 | /* expect it to work, without an error: */ | |
1243 | pkey_assert(ret != -1); | |
1244 | /* Now access from the current task, and expect NO exception: */ | |
1245 | peek_result = read_ptr(plain_ptr); | |
99478798 | 1246 | do_not_expect_pk_fault("read plain pointer after ptrace"); |
5f23f6d0 DH |
1247 | |
1248 | ret = ptrace(PTRACE_DETACH, child_pid, ignored, 0); | |
1249 | pkey_assert(ret != -1); | |
1250 | ||
1251 | ret = kill(child_pid, SIGKILL); | |
1252 | pkey_assert(ret != -1); | |
1253 | ||
1254 | wait(&status); | |
1255 | ||
1256 | free(plain_ptr_unaligned); | |
1257 | } | |
1258 | ||
4b84fd7c | 1259 | void *get_pointer_to_instructions(void) |
5f23f6d0 DH |
1260 | { |
1261 | void *p1; | |
5f23f6d0 DH |
1262 | |
1263 | p1 = ALIGN_PTR_UP(&lots_o_noops_around_write, PAGE_SIZE); | |
1264 | dprintf3("&lots_o_noops: %p\n", &lots_o_noops_around_write); | |
1265 | /* lots_o_noops_around_write should be page-aligned already */ | |
1266 | assert(p1 == &lots_o_noops_around_write); | |
1267 | ||
1268 | /* Point 'p1' at the *second* page of the function: */ | |
1269 | p1 += PAGE_SIZE; | |
1270 | ||
4b84fd7c DH |
1271 | /* |
1272 | * Try to ensure we fault this in on next touch to ensure | |
1273 | * we get an instruction fault as opposed to a data one | |
1274 | */ | |
5f23f6d0 | 1275 | madvise(p1, PAGE_SIZE, MADV_DONTNEED); |
4b84fd7c DH |
1276 | |
1277 | return p1; | |
1278 | } | |
1279 | ||
1280 | void test_executing_on_unreadable_memory(int *ptr, u16 pkey) | |
1281 | { | |
1282 | void *p1; | |
1283 | int scratch; | |
1284 | int ptr_contents; | |
1285 | int ret; | |
1286 | ||
1287 | p1 = get_pointer_to_instructions(); | |
5f23f6d0 DH |
1288 | lots_o_noops_around_write(&scratch); |
1289 | ptr_contents = read_ptr(p1); | |
1290 | dprintf2("ptr (%p) contents@%d: %x\n", p1, __LINE__, ptr_contents); | |
1291 | ||
1292 | ret = mprotect_pkey(p1, PAGE_SIZE, PROT_EXEC, (u64)pkey); | |
1293 | pkey_assert(!ret); | |
1294 | pkey_access_deny(pkey); | |
1295 | ||
1296 | dprintf2("pkru: %x\n", rdpkru()); | |
1297 | ||
1298 | /* | |
1299 | * Make sure this is an *instruction* fault | |
1300 | */ | |
1301 | madvise(p1, PAGE_SIZE, MADV_DONTNEED); | |
1302 | lots_o_noops_around_write(&scratch); | |
99478798 | 1303 | do_not_expect_pk_fault("executing on PROT_EXEC memory"); |
5f23f6d0 DH |
1304 | ptr_contents = read_ptr(p1); |
1305 | dprintf2("ptr (%p) contents@%d: %x\n", p1, __LINE__, ptr_contents); | |
1306 | expected_pk_fault(pkey); | |
1307 | } | |
1308 | ||
a0c68652 DH |
1309 | void test_implicit_mprotect_exec_only_memory(int *ptr, u16 pkey) |
1310 | { | |
1311 | void *p1; | |
1312 | int scratch; | |
1313 | int ptr_contents; | |
1314 | int ret; | |
1315 | ||
1316 | dprintf1("%s() start\n", __func__); | |
1317 | ||
1318 | p1 = get_pointer_to_instructions(); | |
1319 | lots_o_noops_around_write(&scratch); | |
1320 | ptr_contents = read_ptr(p1); | |
1321 | dprintf2("ptr (%p) contents@%d: %x\n", p1, __LINE__, ptr_contents); | |
1322 | ||
1323 | /* Use a *normal* mprotect(), not mprotect_pkey(): */ | |
1324 | ret = mprotect(p1, PAGE_SIZE, PROT_EXEC); | |
1325 | pkey_assert(!ret); | |
1326 | ||
1327 | dprintf2("pkru: %x\n", rdpkru()); | |
1328 | ||
1329 | /* Make sure this is an *instruction* fault */ | |
1330 | madvise(p1, PAGE_SIZE, MADV_DONTNEED); | |
1331 | lots_o_noops_around_write(&scratch); | |
1332 | do_not_expect_pk_fault("executing on PROT_EXEC memory"); | |
1333 | ptr_contents = read_ptr(p1); | |
1334 | dprintf2("ptr (%p) contents@%d: %x\n", p1, __LINE__, ptr_contents); | |
1335 | expected_pk_fault(UNKNOWN_PKEY); | |
1336 | ||
1337 | /* | |
1338 | * Put the memory back to non-PROT_EXEC. Should clear the | |
1339 | * exec-only pkey off the VMA and allow it to be readable | |
1340 | * again. Go to PROT_NONE first to check for a kernel bug | |
1341 | * that did not clear the pkey when doing PROT_NONE. | |
1342 | */ | |
1343 | ret = mprotect(p1, PAGE_SIZE, PROT_NONE); | |
1344 | pkey_assert(!ret); | |
1345 | ||
1346 | ret = mprotect(p1, PAGE_SIZE, PROT_READ|PROT_EXEC); | |
1347 | pkey_assert(!ret); | |
1348 | ptr_contents = read_ptr(p1); | |
1349 | do_not_expect_pk_fault("plain read on recently PROT_EXEC area"); | |
1350 | } | |
1351 | ||
5f23f6d0 DH |
1352 | void test_mprotect_pkey_on_unsupported_cpu(int *ptr, u16 pkey) |
1353 | { | |
1354 | int size = PAGE_SIZE; | |
1355 | int sret; | |
1356 | ||
1357 | if (cpu_has_pku()) { | |
1358 | dprintf1("SKIP: %s: no CPU support\n", __func__); | |
1359 | return; | |
1360 | } | |
1361 | ||
1362 | sret = syscall(SYS_mprotect_key, ptr, size, PROT_READ, pkey); | |
1363 | pkey_assert(sret < 0); | |
1364 | } | |
1365 | ||
1366 | void (*pkey_tests[])(int *ptr, u16 pkey) = { | |
1367 | test_read_of_write_disabled_region, | |
1368 | test_read_of_access_disabled_region, | |
1369 | test_write_of_write_disabled_region, | |
1370 | test_write_of_access_disabled_region, | |
1371 | test_kernel_write_of_access_disabled_region, | |
1372 | test_kernel_write_of_write_disabled_region, | |
1373 | test_kernel_gup_of_access_disabled_region, | |
1374 | test_kernel_gup_write_to_write_disabled_region, | |
1375 | test_executing_on_unreadable_memory, | |
a0c68652 | 1376 | test_implicit_mprotect_exec_only_memory, |
5f23f6d0 DH |
1377 | test_ptrace_of_child, |
1378 | test_pkey_syscalls_on_non_allocated_pkey, | |
1379 | test_pkey_syscalls_bad_args, | |
1380 | test_pkey_alloc_exhaust, | |
1381 | }; | |
1382 | ||
1383 | void run_tests_once(void) | |
1384 | { | |
1385 | int *ptr; | |
1386 | int prot = PROT_READ|PROT_WRITE; | |
1387 | ||
1388 | for (test_nr = 0; test_nr < ARRAY_SIZE(pkey_tests); test_nr++) { | |
1389 | int pkey; | |
1390 | int orig_pkru_faults = pkru_faults; | |
1391 | ||
1392 | dprintf1("======================\n"); | |
1393 | dprintf1("test %d preparing...\n", test_nr); | |
1394 | ||
1395 | tracing_on(); | |
1396 | pkey = alloc_random_pkey(); | |
1397 | dprintf1("test %d starting with pkey: %d\n", test_nr, pkey); | |
1398 | ptr = malloc_pkey(PAGE_SIZE, prot, pkey); | |
1399 | dprintf1("test %d starting...\n", test_nr); | |
1400 | pkey_tests[test_nr](ptr, pkey); | |
1401 | dprintf1("freeing test memory: %p\n", ptr); | |
1402 | free_pkey_malloc(ptr); | |
1403 | sys_pkey_free(pkey); | |
1404 | ||
1405 | dprintf1("pkru_faults: %d\n", pkru_faults); | |
1406 | dprintf1("orig_pkru_faults: %d\n", orig_pkru_faults); | |
1407 | ||
1408 | tracing_off(); | |
1409 | close_test_fds(); | |
1410 | ||
7738789f | 1411 | printf("test %2d PASSED (iteration %d)\n", test_nr, iteration_nr); |
5f23f6d0 DH |
1412 | dprintf1("======================\n\n"); |
1413 | } | |
1414 | iteration_nr++; | |
1415 | } | |
1416 | ||
1417 | void pkey_setup_shadow(void) | |
1418 | { | |
1419 | shadow_pkru = __rdpkru(); | |
1420 | } | |
1421 | ||
1422 | int main(void) | |
1423 | { | |
1424 | int nr_iterations = 22; | |
1425 | ||
1426 | setup_handlers(); | |
1427 | ||
1428 | printf("has pku: %d\n", cpu_has_pku()); | |
1429 | ||
1430 | if (!cpu_has_pku()) { | |
1431 | int size = PAGE_SIZE; | |
1432 | int *ptr; | |
1433 | ||
1434 | printf("running PKEY tests for unsupported CPU/OS\n"); | |
1435 | ||
1436 | ptr = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); | |
1437 | assert(ptr != (void *)-1); | |
1438 | test_mprotect_pkey_on_unsupported_cpu(ptr, 1); | |
1439 | exit(0); | |
1440 | } | |
1441 | ||
1442 | pkey_setup_shadow(); | |
1443 | printf("startup pkru: %x\n", rdpkru()); | |
1444 | setup_hugetlbfs(); | |
1445 | ||
1446 | while (nr_iterations-- > 0) | |
1447 | run_tests_once(); | |
1448 | ||
1449 | printf("done (all tests OK)\n"); | |
1450 | return 0; | |
1451 | } |