2 * Tests x86 Memory Protection Keys (see Documentation/x86/protection-keys.txt)
4 * There are examples in here of:
5 * * how to set protection keys on memory
6 * * how to set/clear bits in PKRU (the rights register)
7 * * how to handle SEGV_PKRU signals and extract pkey-relevant
8 * information from the siginfo
11 * make sure KSM and KSM COW breaking works
12 * prefault pages in at malloc, or not
13 * protect MPX bounds tables with protection keys?
14 * make sure VMA splitting/merging is working correctly
15 * OOMs can destroy mm->mmap (see exit_mmap()), so make sure it is immune to pkeys
16 * look for pkey "leaks" where it is still set on a VMA but "freed" back to the kernel
17 * do a plain mprotect() to a mprotect_pkey() area and make sure the pkey sticks
20 * gcc -o protection_keys -O2 -g -std=gnu99 -pthread -Wall protection_keys.c -lrt -ldl -lm
21 * gcc -m32 -o protection_keys_32 -O2 -g -std=gnu99 -pthread -Wall protection_keys.c -lrt -ldl -lm
25 #include <linux/futex.h>
28 #include <sys/syscall.h>
38 #include <sys/types.h>
43 #include <sys/ptrace.h>
46 #include "pkey-helpers.h"
51 unsigned int shadow_pkru;
53 #define HPAGE_SIZE (1UL<<21)
54 #define ARRAY_SIZE(x) (sizeof(x) / sizeof(*(x)))
55 #define ALIGN_UP(x, align_to) (((x) + ((align_to)-1)) & ~((align_to)-1))
56 #define ALIGN_DOWN(x, align_to) ((x) & ~((align_to)-1))
57 #define ALIGN_PTR_UP(p, ptr_align_to) ((typeof(p))ALIGN_UP((unsigned long)(p), ptr_align_to))
58 #define ALIGN_PTR_DOWN(p, ptr_align_to) ((typeof(p))ALIGN_DOWN((unsigned long)(p), ptr_align_to))
59 #define __stringify_1(x...) #x
60 #define __stringify(x...) __stringify_1(x)
62 #define PTR_ERR_ENOTSUP ((void *)-ENOTSUP)
65 char dprint_in_signal_buffer[DPRINT_IN_SIGNAL_BUF_SIZE];
67 extern void abort_hooks(void);
68 #define pkey_assert(condition) do { \
70 dprintf0("assert() at %s::%d test_nr: %d iteration: %d\n", \
72 test_nr, iteration_nr); \
73 dprintf0("errno at assert: %d", errno); \
78 #define raw_assert(cond) assert(cond)
80 void cat_into_file(char *str, char *file)
82 int fd = open(file, O_RDWR);
85 dprintf2("%s(): writing '%s' to '%s'\n", __func__, str, file);
87 * these need to be raw because they are called under
91 ret = write(fd, str, strlen(str));
92 if (ret != strlen(str)) {
93 perror("write to file failed");
94 fprintf(stderr, "filename: '%s' str: '%s'\n", file, str);
100 #if CONTROL_TRACING > 0
101 static int warned_tracing;
102 int tracing_root_ok(void)
104 if (geteuid() != 0) {
106 fprintf(stderr, "WARNING: not run as root, "
107 "can not do tracing control\n");
115 void tracing_on(void)
117 #if CONTROL_TRACING > 0
118 #define TRACEDIR "/sys/kernel/debug/tracing"
121 if (!tracing_root_ok())
124 sprintf(pidstr, "%d", getpid());
125 cat_into_file("0", TRACEDIR "/tracing_on");
126 cat_into_file("\n", TRACEDIR "/trace");
128 cat_into_file("function_graph", TRACEDIR "/current_tracer");
129 cat_into_file("1", TRACEDIR "/options/funcgraph-proc");
131 cat_into_file("nop", TRACEDIR "/current_tracer");
133 cat_into_file(pidstr, TRACEDIR "/set_ftrace_pid");
134 cat_into_file("1", TRACEDIR "/tracing_on");
135 dprintf1("enabled tracing\n");
139 void tracing_off(void)
141 #if CONTROL_TRACING > 0
142 if (!tracing_root_ok())
144 cat_into_file("0", "/sys/kernel/debug/tracing/tracing_on");
148 void abort_hooks(void)
150 fprintf(stderr, "running %s()...\n", __func__);
152 #ifdef SLEEP_ON_ABORT
153 sleep(SLEEP_ON_ABORT);
157 static inline void __page_o_noops(void)
159 /* 8-bytes of instruction * 512 bytes = 1 page */
160 asm(".rept 512 ; nopl 0x7eeeeeee(%eax) ; .endr");
164 * This attempts to have roughly a page of instructions followed by a few
165 * instructions that do a write, and another page of instructions. That
166 * way, we are pretty sure that the write is in the second page of
167 * instructions and has at least a page of padding behind it.
169 * *That* lets us be sure to madvise() away the write instruction, which
170 * will then fault, which makes sure that the fault code handles
171 * execute-only memory properly.
173 __attribute__((__aligned__(PAGE_SIZE)))
174 void lots_o_noops_around_write(int *write_to_me)
176 dprintf3("running %s()\n", __func__);
178 /* Assume this happens in the second page of instructions: */
179 *write_to_me = __LINE__;
180 /* pad out by another page: */
182 dprintf3("%s() done\n", __func__);
185 /* Define some kernel-like types */
193 #ifndef SYS_mprotect_key
194 # define SYS_mprotect_key 380
196 #ifndef SYS_pkey_alloc
197 # define SYS_pkey_alloc 381
198 # define SYS_pkey_free 382
200 #define REG_IP_IDX REG_EIP
201 #define si_pkey_offset 0x14
205 #ifndef SYS_mprotect_key
206 # define SYS_mprotect_key 329
208 #ifndef SYS_pkey_alloc
209 # define SYS_pkey_alloc 330
210 # define SYS_pkey_free 331
212 #define REG_IP_IDX REG_RIP
213 #define si_pkey_offset 0x20
217 void dump_mem(void *dumpme, int len_bytes)
219 char *c = (void *)dumpme;
222 for (i = 0; i < len_bytes; i += sizeof(u64)) {
223 u64 *ptr = (u64 *)(c + i);
224 dprintf1("dump[%03d][@%p]: %016jx\n", i, ptr, *ptr);
228 #define SEGV_BNDERR 3 /* failed address bound checks */
229 #define SEGV_PKUERR 4
231 static char *si_code_str(int si_code)
233 if (si_code == SEGV_MAPERR)
234 return "SEGV_MAPERR";
235 if (si_code == SEGV_ACCERR)
236 return "SEGV_ACCERR";
237 if (si_code == SEGV_BNDERR)
238 return "SEGV_BNDERR";
239 if (si_code == SEGV_PKUERR)
240 return "SEGV_PKUERR";
245 int last_si_pkey = -1;
246 void signal_handler(int signum, siginfo_t *si, void *vucontext)
248 ucontext_t *uctxt = vucontext;
258 dprint_in_signal = 1;
259 dprintf1(">>>>===============SIGSEGV============================\n");
260 dprintf1("%s()::%d, pkru: 0x%x shadow: %x\n", __func__, __LINE__,
261 __rdpkru(), shadow_pkru);
263 trapno = uctxt->uc_mcontext.gregs[REG_TRAPNO];
264 ip = uctxt->uc_mcontext.gregs[REG_IP_IDX];
265 fpregset = uctxt->uc_mcontext.fpregs;
266 fpregs = (void *)fpregset;
268 dprintf2("%s() trapno: %d ip: 0x%lx info->si_code: %s/%d\n", __func__,
269 trapno, ip, si_code_str(si->si_code), si->si_code);
272 * 32-bit has some extra padding so that userspace can tell whether
273 * the XSTATE header is present in addition to the "legacy" FPU
274 * state. We just assume that it is here.
278 pkru_offset = pkru_xstate_offset();
279 pkru_ptr = (void *)(&fpregs[pkru_offset]);
281 dprintf1("siginfo: %p\n", si);
282 dprintf1(" fpregs: %p\n", fpregs);
284 * If we got a PKRU fault, we *HAVE* to have at least one bit set in
287 dprintf1("pkru_xstate_offset: %d\n", pkru_xstate_offset());
289 dump_mem(pkru_ptr - 128, 256);
290 pkey_assert(*pkru_ptr);
292 si_pkey_ptr = (u32 *)(((u8 *)si) + si_pkey_offset);
293 dprintf1("si_pkey_ptr: %p\n", si_pkey_ptr);
294 dump_mem(si_pkey_ptr - 8, 24);
295 siginfo_pkey = *si_pkey_ptr;
296 pkey_assert(siginfo_pkey < NR_PKEYS);
297 last_si_pkey = siginfo_pkey;
299 if ((si->si_code == SEGV_MAPERR) ||
300 (si->si_code == SEGV_ACCERR) ||
301 (si->si_code == SEGV_BNDERR)) {
302 printf("non-PK si_code, exiting...\n");
306 dprintf1("signal pkru from xsave: %08x\n", *pkru_ptr);
307 /* need __rdpkru() version so we do not do shadow_pkru checking */
308 dprintf1("signal pkru from pkru: %08x\n", __rdpkru());
309 dprintf1("pkey from siginfo: %jx\n", siginfo_pkey);
310 *(u64 *)pkru_ptr = 0x00000000;
311 dprintf1("WARNING: set PRKU=0 to allow faulting instruction to continue\n");
313 dprintf1("<<<<==================================================\n");
317 "ERROR: In signal handler, page fault, trapno = %d, ip = %016lx\n",
319 fprintf(stderr, "si_addr %p\n", si->si_addr);
320 fprintf(stderr, "REG_ERR: %lx\n",
321 (unsigned long)uctxt->uc_mcontext.gregs[REG_ERR]);
324 fprintf(stderr, "unexpected trap %d! at 0x%lx\n", trapno, ip);
325 fprintf(stderr, "si_addr %p\n", si->si_addr);
326 fprintf(stderr, "REG_ERR: %lx\n",
327 (unsigned long)uctxt->uc_mcontext.gregs[REG_ERR]);
330 dprint_in_signal = 0;
333 int wait_all_children(void)
336 return waitpid(-1, &status, 0);
341 dprint_in_signal = 1;
342 dprintf2("[%d] SIGCHLD: %d\n", getpid(), x);
343 dprint_in_signal = 0;
346 void setup_sigsegv_handler(void)
349 struct sigaction newact;
350 struct sigaction oldact;
352 /* #PF is mapped to sigsegv */
353 int signum = SIGSEGV;
355 newact.sa_handler = 0;
356 newact.sa_sigaction = signal_handler;
358 /*sigset_t - signals to block while in the handler */
359 /* get the old signal mask. */
360 rs = sigprocmask(SIG_SETMASK, 0, &newact.sa_mask);
361 pkey_assert(rs == 0);
363 /* call sa_sigaction, not sa_handler*/
364 newact.sa_flags = SA_SIGINFO;
366 newact.sa_restorer = 0; /* void(*)(), obsolete */
367 r = sigaction(signum, &newact, &oldact);
368 r = sigaction(SIGALRM, &newact, &oldact);
372 void setup_handlers(void)
374 signal(SIGCHLD, &sig_chld);
375 setup_sigsegv_handler();
378 pid_t fork_lazy_child(void)
383 pkey_assert(forkret >= 0);
384 dprintf3("[%d] fork() ret: %d\n", getpid(), forkret);
389 dprintf1("child sleeping...\n");
396 #define PKEY_DISABLE_ACCESS 0x1
397 #define PKEY_DISABLE_WRITE 0x2
399 u32 pkey_get(int pkey, unsigned long flags)
401 u32 mask = (PKEY_DISABLE_ACCESS|PKEY_DISABLE_WRITE);
402 u32 pkru = __rdpkru();
406 dprintf1("%s(pkey=%d, flags=%lx) = %x / %d\n",
407 __func__, pkey, flags, 0, 0);
408 dprintf2("%s() raw pkru: %x\n", __func__, pkru);
410 shifted_pkru = (pkru >> (pkey * PKRU_BITS_PER_PKEY));
411 dprintf2("%s() shifted_pkru: %x\n", __func__, shifted_pkru);
412 masked_pkru = shifted_pkru & mask;
413 dprintf2("%s() masked pkru: %x\n", __func__, masked_pkru);
415 * shift down the relevant bits to the lowest two, then
416 * mask off all the other high bits.
421 int pkey_set(int pkey, unsigned long rights, unsigned long flags)
423 u32 mask = (PKEY_DISABLE_ACCESS|PKEY_DISABLE_WRITE);
424 u32 old_pkru = __rdpkru();
427 /* make sure that 'rights' only contains the bits we expect: */
428 assert(!(rights & ~mask));
432 /* mask out bits from pkey in old value: */
433 new_pkru &= ~(mask << (pkey * PKRU_BITS_PER_PKEY));
434 /* OR in new bits for pkey: */
435 new_pkru |= (rights << (pkey * PKRU_BITS_PER_PKEY));
439 dprintf3("%s(pkey=%d, rights=%lx, flags=%lx) = %x pkru now: %x old_pkru: %x\n",
440 __func__, pkey, rights, flags, 0, __rdpkru(), old_pkru);
444 void pkey_disable_set(int pkey, int flags)
446 unsigned long syscall_flags = 0;
451 dprintf1("START->%s(%d, 0x%x)\n", __func__,
453 pkey_assert(flags & (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE));
455 pkey_rights = pkey_get(pkey, syscall_flags);
457 dprintf1("%s(%d) pkey_get(%d): %x\n", __func__,
458 pkey, pkey, pkey_rights);
459 pkey_assert(pkey_rights >= 0);
461 pkey_rights |= flags;
463 ret = pkey_set(pkey, pkey_rights, syscall_flags);
465 /*pkru and flags have the same format */
466 shadow_pkru |= flags << (pkey * 2);
467 dprintf1("%s(%d) shadow: 0x%x\n", __func__, pkey, shadow_pkru);
469 pkey_assert(ret >= 0);
471 pkey_rights = pkey_get(pkey, syscall_flags);
472 dprintf1("%s(%d) pkey_get(%d): %x\n", __func__,
473 pkey, pkey, pkey_rights);
475 dprintf1("%s(%d) pkru: 0x%x\n", __func__, pkey, rdpkru());
477 pkey_assert(rdpkru() > orig_pkru);
478 dprintf1("END<---%s(%d, 0x%x)\n", __func__,
482 void pkey_disable_clear(int pkey, int flags)
484 unsigned long syscall_flags = 0;
486 int pkey_rights = pkey_get(pkey, syscall_flags);
487 u32 orig_pkru = rdpkru();
489 pkey_assert(flags & (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE));
491 dprintf1("%s(%d) pkey_get(%d): %x\n", __func__,
492 pkey, pkey, pkey_rights);
493 pkey_assert(pkey_rights >= 0);
495 pkey_rights |= flags;
497 ret = pkey_set(pkey, pkey_rights, 0);
498 /* pkru and flags have the same format */
499 shadow_pkru &= ~(flags << (pkey * 2));
500 pkey_assert(ret >= 0);
502 pkey_rights = pkey_get(pkey, syscall_flags);
503 dprintf1("%s(%d) pkey_get(%d): %x\n", __func__,
504 pkey, pkey, pkey_rights);
506 dprintf1("%s(%d) pkru: 0x%x\n", __func__, pkey, rdpkru());
508 assert(rdpkru() > orig_pkru);
511 void pkey_write_allow(int pkey)
513 pkey_disable_clear(pkey, PKEY_DISABLE_WRITE);
515 void pkey_write_deny(int pkey)
517 pkey_disable_set(pkey, PKEY_DISABLE_WRITE);
519 void pkey_access_allow(int pkey)
521 pkey_disable_clear(pkey, PKEY_DISABLE_ACCESS);
523 void pkey_access_deny(int pkey)
525 pkey_disable_set(pkey, PKEY_DISABLE_ACCESS);
528 int sys_mprotect_pkey(void *ptr, size_t size, unsigned long orig_prot,
533 dprintf2("%s(0x%p, %zx, prot=%lx, pkey=%lx)\n", __func__,
534 ptr, size, orig_prot, pkey);
537 sret = syscall(SYS_mprotect_key, ptr, size, orig_prot, pkey);
539 dprintf2("SYS_mprotect_key sret: %d\n", sret);
540 dprintf2("SYS_mprotect_key prot: 0x%lx\n", orig_prot);
541 dprintf2("SYS_mprotect_key failed, errno: %d\n", errno);
542 if (DEBUG_LEVEL >= 2)
543 perror("SYS_mprotect_pkey");
548 int sys_pkey_alloc(unsigned long flags, unsigned long init_val)
550 int ret = syscall(SYS_pkey_alloc, flags, init_val);
551 dprintf1("%s(flags=%lx, init_val=%lx) syscall ret: %d errno: %d\n",
552 __func__, flags, init_val, ret, errno);
559 unsigned long init_val = 0x0;
561 dprintf1("alloc_pkey()::%d, pkru: 0x%x shadow: %x\n",
562 __LINE__, __rdpkru(), shadow_pkru);
563 ret = sys_pkey_alloc(0, init_val);
565 * pkey_alloc() sets PKRU, so we need to reflect it in
568 dprintf4("alloc_pkey()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n",
569 __LINE__, ret, __rdpkru(), shadow_pkru);
571 /* clear both the bits: */
572 shadow_pkru &= ~(0x3 << (ret * 2));
573 dprintf4("alloc_pkey()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n",
574 __LINE__, ret, __rdpkru(), shadow_pkru);
576 * move the new state in from init_val
577 * (remember, we cheated and init_val == pkru format)
579 shadow_pkru |= (init_val << (ret * 2));
581 dprintf4("alloc_pkey()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n",
582 __LINE__, ret, __rdpkru(), shadow_pkru);
583 dprintf1("alloc_pkey()::%d errno: %d\n", __LINE__, errno);
584 /* for shadow checking: */
586 dprintf4("alloc_pkey()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n",
587 __LINE__, ret, __rdpkru(), shadow_pkru);
591 int sys_pkey_free(unsigned long pkey)
593 int ret = syscall(SYS_pkey_free, pkey);
594 dprintf1("%s(pkey=%ld) syscall ret: %d\n", __func__, pkey, ret);
599 * I had a bug where pkey bits could be set by mprotect() but
600 * not cleared. This ensures we get lots of random bit sets
601 * and clears on the vma and pte pkey bits.
603 int alloc_random_pkey(void)
605 int max_nr_pkey_allocs;
608 int alloced_pkeys[NR_PKEYS];
611 memset(alloced_pkeys, 0, sizeof(alloced_pkeys));
613 /* allocate every possible key and make a note of which ones we got */
614 max_nr_pkey_allocs = NR_PKEYS;
615 for (i = 0; i < max_nr_pkey_allocs; i++) {
616 int new_pkey = alloc_pkey();
619 alloced_pkeys[nr_alloced++] = new_pkey;
622 pkey_assert(nr_alloced > 0);
623 /* select a random one out of the allocated ones */
624 random_index = rand() % nr_alloced;
625 ret = alloced_pkeys[random_index];
626 /* now zero it out so we don't free it next */
627 alloced_pkeys[random_index] = 0;
629 /* go through the allocated ones that we did not want and free them */
630 for (i = 0; i < nr_alloced; i++) {
632 if (!alloced_pkeys[i])
634 free_ret = sys_pkey_free(alloced_pkeys[i]);
635 pkey_assert(!free_ret);
637 dprintf1("%s()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", __func__,
638 __LINE__, ret, __rdpkru(), shadow_pkru);
642 int mprotect_pkey(void *ptr, size_t size, unsigned long orig_prot,
645 int nr_iterations = random() % 100;
649 int rpkey = alloc_random_pkey();
650 ret = sys_mprotect_pkey(ptr, size, orig_prot, pkey);
651 dprintf1("sys_mprotect_pkey(%p, %zx, prot=0x%lx, pkey=%ld) ret: %d\n",
652 ptr, size, orig_prot, pkey, ret);
653 if (nr_iterations-- < 0)
656 dprintf1("%s()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", __func__,
657 __LINE__, ret, __rdpkru(), shadow_pkru);
658 sys_pkey_free(rpkey);
659 dprintf1("%s()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", __func__,
660 __LINE__, ret, __rdpkru(), shadow_pkru);
662 pkey_assert(pkey < NR_PKEYS);
664 ret = sys_mprotect_pkey(ptr, size, orig_prot, pkey);
665 dprintf1("mprotect_pkey(%p, %zx, prot=0x%lx, pkey=%ld) ret: %d\n",
666 ptr, size, orig_prot, pkey, ret);
668 dprintf1("%s()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", __func__,
669 __LINE__, ret, __rdpkru(), shadow_pkru);
673 struct pkey_malloc_record {
677 struct pkey_malloc_record *pkey_malloc_records;
678 long nr_pkey_malloc_records;
679 void record_pkey_malloc(void *ptr, long size)
682 struct pkey_malloc_record *rec = NULL;
684 for (i = 0; i < nr_pkey_malloc_records; i++) {
685 rec = &pkey_malloc_records[i];
686 /* find a free record */
691 /* every record is full */
692 size_t old_nr_records = nr_pkey_malloc_records;
693 size_t new_nr_records = (nr_pkey_malloc_records * 2 + 1);
694 size_t new_size = new_nr_records * sizeof(struct pkey_malloc_record);
695 dprintf2("new_nr_records: %zd\n", new_nr_records);
696 dprintf2("new_size: %zd\n", new_size);
697 pkey_malloc_records = realloc(pkey_malloc_records, new_size);
698 pkey_assert(pkey_malloc_records != NULL);
699 rec = &pkey_malloc_records[nr_pkey_malloc_records];
701 * realloc() does not initialize memory, so zero it from
702 * the first new record all the way to the end.
704 for (i = 0; i < new_nr_records - old_nr_records; i++)
705 memset(rec + i, 0, sizeof(*rec));
707 dprintf3("filling malloc record[%d/%p]: {%p, %ld}\n",
708 (int)(rec - pkey_malloc_records), rec, ptr, size);
711 nr_pkey_malloc_records++;
714 void free_pkey_malloc(void *ptr)
718 dprintf3("%s(%p)\n", __func__, ptr);
719 for (i = 0; i < nr_pkey_malloc_records; i++) {
720 struct pkey_malloc_record *rec = &pkey_malloc_records[i];
721 dprintf4("looking for ptr %p at record[%ld/%p]: {%p, %ld}\n",
722 ptr, i, rec, rec->ptr, rec->size);
723 if ((ptr < rec->ptr) ||
724 (ptr >= rec->ptr + rec->size))
727 dprintf3("found ptr %p at record[%ld/%p]: {%p, %ld}\n",
728 ptr, i, rec, rec->ptr, rec->size);
729 nr_pkey_malloc_records--;
730 ret = munmap(rec->ptr, rec->size);
731 dprintf3("munmap ret: %d\n", ret);
733 dprintf3("clearing rec->ptr, rec: %p\n", rec);
735 dprintf3("done clearing rec->ptr, rec: %p\n", rec);
742 void *malloc_pkey_with_mprotect(long size, int prot, u16 pkey)
748 dprintf1("doing %s(size=%ld, prot=0x%x, pkey=%d)\n", __func__,
750 pkey_assert(pkey < NR_PKEYS);
751 ptr = mmap(NULL, size, prot, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
752 pkey_assert(ptr != (void *)-1);
753 ret = mprotect_pkey((void *)ptr, PAGE_SIZE, prot, pkey);
755 record_pkey_malloc(ptr, size);
758 dprintf1("%s() for pkey %d @ %p\n", __func__, pkey, ptr);
762 void *malloc_pkey_anon_huge(long size, int prot, u16 pkey)
767 dprintf1("doing %s(size=%ld, prot=0x%x, pkey=%d)\n", __func__,
770 * Guarantee we can fit at least one huge page in the resulting
771 * allocation by allocating space for 2:
773 size = ALIGN_UP(size, HPAGE_SIZE * 2);
774 ptr = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
775 pkey_assert(ptr != (void *)-1);
776 record_pkey_malloc(ptr, size);
777 mprotect_pkey(ptr, size, prot, pkey);
779 dprintf1("unaligned ptr: %p\n", ptr);
780 ptr = ALIGN_PTR_UP(ptr, HPAGE_SIZE);
781 dprintf1(" aligned ptr: %p\n", ptr);
782 ret = madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE);
783 dprintf1("MADV_HUGEPAGE ret: %d\n", ret);
784 ret = madvise(ptr, HPAGE_SIZE, MADV_WILLNEED);
785 dprintf1("MADV_WILLNEED ret: %d\n", ret);
786 memset(ptr, 0, HPAGE_SIZE);
788 dprintf1("mmap()'d thp for pkey %d @ %p\n", pkey, ptr);
792 int hugetlb_setup_ok;
793 #define GET_NR_HUGE_PAGES 10
794 void setup_hugetlbfs(void)
798 int validated_nr_pages;
802 if (geteuid() != 0) {
803 fprintf(stderr, "WARNING: not run as root, can not do hugetlb test\n");
807 cat_into_file(__stringify(GET_NR_HUGE_PAGES), "/proc/sys/vm/nr_hugepages");
810 * Now go make sure that we got the pages and that they
811 * are 2M pages. Someone might have made 1G the default.
813 fd = open("/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages", O_RDONLY);
815 perror("opening sysfs 2M hugetlb config");
819 /* -1 to guarantee leaving the trailing \0 */
820 err = read(fd, buf, sizeof(buf)-1);
823 perror("reading sysfs 2M hugetlb config");
827 if (atoi(buf) != GET_NR_HUGE_PAGES) {
828 fprintf(stderr, "could not confirm 2M pages, got: '%s' expected %d\n",
829 buf, GET_NR_HUGE_PAGES);
833 hugetlb_setup_ok = 1;
836 void *malloc_pkey_hugetlb(long size, int prot, u16 pkey)
839 int flags = MAP_ANONYMOUS|MAP_PRIVATE|MAP_HUGETLB;
841 if (!hugetlb_setup_ok)
842 return PTR_ERR_ENOTSUP;
844 dprintf1("doing %s(%ld, %x, %x)\n", __func__, size, prot, pkey);
845 size = ALIGN_UP(size, HPAGE_SIZE * 2);
846 pkey_assert(pkey < NR_PKEYS);
847 ptr = mmap(NULL, size, PROT_NONE, flags, -1, 0);
848 pkey_assert(ptr != (void *)-1);
849 mprotect_pkey(ptr, size, prot, pkey);
851 record_pkey_malloc(ptr, size);
853 dprintf1("mmap()'d hugetlbfs for pkey %d @ %p\n", pkey, ptr);
857 void *malloc_pkey_mmap_dax(long size, int prot, u16 pkey)
862 dprintf1("doing %s(size=%ld, prot=0x%x, pkey=%d)\n", __func__,
864 pkey_assert(pkey < NR_PKEYS);
865 fd = open("/dax/foo", O_RDWR);
866 pkey_assert(fd >= 0);
868 ptr = mmap(0, size, prot, MAP_SHARED, fd, 0);
869 pkey_assert(ptr != (void *)-1);
871 mprotect_pkey(ptr, size, prot, pkey);
873 record_pkey_malloc(ptr, size);
875 dprintf1("mmap()'d for pkey %d @ %p\n", pkey, ptr);
880 void *(*pkey_malloc[])(long size, int prot, u16 pkey) = {
882 malloc_pkey_with_mprotect,
883 malloc_pkey_anon_huge,
885 /* can not do direct with the pkey_mprotect() API:
886 malloc_pkey_mmap_direct,
887 malloc_pkey_mmap_dax,
891 void *malloc_pkey(long size, int prot, u16 pkey)
894 static int malloc_type;
895 int nr_malloc_types = ARRAY_SIZE(pkey_malloc);
897 pkey_assert(pkey < NR_PKEYS);
900 pkey_assert(malloc_type < nr_malloc_types);
902 ret = pkey_malloc[malloc_type](size, prot, pkey);
903 pkey_assert(ret != (void *)-1);
906 if (malloc_type >= nr_malloc_types)
907 malloc_type = (random()%nr_malloc_types);
909 /* try again if the malloc_type we tried is unsupported */
910 if (ret == PTR_ERR_ENOTSUP)
916 dprintf3("%s(%ld, prot=%x, pkey=%x) returning: %p\n", __func__,
917 size, prot, pkey, ret);
921 int last_pkru_faults;
922 void expected_pk_fault(int pkey)
924 dprintf2("%s(): last_pkru_faults: %d pkru_faults: %d\n",
925 __func__, last_pkru_faults, pkru_faults);
926 dprintf2("%s(%d): last_si_pkey: %d\n", __func__, pkey, last_si_pkey);
927 pkey_assert(last_pkru_faults + 1 == pkru_faults);
928 pkey_assert(last_si_pkey == pkey);
930 * The signal handler shold have cleared out PKRU to let the
931 * test program continue. We now have to restore it.
936 __wrpkru(shadow_pkru);
937 dprintf1("%s() set PKRU=%x to restore state after signal nuked it\n",
938 __func__, shadow_pkru);
939 last_pkru_faults = pkru_faults;
943 void do_not_expect_pk_fault(void)
945 pkey_assert(last_pkru_faults == pkru_faults);
948 int test_fds[10] = { -1 };
950 void __save_test_fd(int fd)
952 pkey_assert(fd >= 0);
953 pkey_assert(nr_test_fds < ARRAY_SIZE(test_fds));
954 test_fds[nr_test_fds] = fd;
958 int get_test_read_fd(void)
960 int test_fd = open("/etc/passwd", O_RDONLY);
961 __save_test_fd(test_fd);
965 void close_test_fds(void)
969 for (i = 0; i < nr_test_fds; i++) {
978 #define barrier() __asm__ __volatile__("": : :"memory")
979 __attribute__((noinline)) int read_ptr(int *ptr)
982 * Keep GCC from optimizing this away somehow
988 void test_read_of_write_disabled_region(int *ptr, u16 pkey)
992 dprintf1("disabling write access to PKEY[1], doing read\n");
993 pkey_write_deny(pkey);
994 ptr_contents = read_ptr(ptr);
995 dprintf1("*ptr: %d\n", ptr_contents);
998 void test_read_of_access_disabled_region(int *ptr, u16 pkey)
1002 dprintf1("disabling access to PKEY[%02d], doing read @ %p\n", pkey, ptr);
1004 pkey_access_deny(pkey);
1005 ptr_contents = read_ptr(ptr);
1006 dprintf1("*ptr: %d\n", ptr_contents);
1007 expected_pk_fault(pkey);
1009 void test_write_of_write_disabled_region(int *ptr, u16 pkey)
1011 dprintf1("disabling write access to PKEY[%02d], doing write\n", pkey);
1012 pkey_write_deny(pkey);
1014 expected_pk_fault(pkey);
1016 void test_write_of_access_disabled_region(int *ptr, u16 pkey)
1018 dprintf1("disabling access to PKEY[%02d], doing write\n", pkey);
1019 pkey_access_deny(pkey);
1021 expected_pk_fault(pkey);
1023 void test_kernel_write_of_access_disabled_region(int *ptr, u16 pkey)
1026 int test_fd = get_test_read_fd();
1028 dprintf1("disabling access to PKEY[%02d], "
1029 "having kernel read() to buffer\n", pkey);
1030 pkey_access_deny(pkey);
1031 ret = read(test_fd, ptr, 1);
1032 dprintf1("read ret: %d\n", ret);
1035 void test_kernel_write_of_write_disabled_region(int *ptr, u16 pkey)
1038 int test_fd = get_test_read_fd();
1040 pkey_write_deny(pkey);
1041 ret = read(test_fd, ptr, 100);
1042 dprintf1("read ret: %d\n", ret);
1043 if (ret < 0 && (DEBUG_LEVEL > 0))
1044 perror("verbose read result (OK for this to be bad)");
1048 void test_kernel_gup_of_access_disabled_region(int *ptr, u16 pkey)
1050 int pipe_ret, vmsplice_ret;
1054 pipe_ret = pipe(pipe_fds);
1056 pkey_assert(pipe_ret == 0);
1057 dprintf1("disabling access to PKEY[%02d], "
1058 "having kernel vmsplice from buffer\n", pkey);
1059 pkey_access_deny(pkey);
1061 iov.iov_len = PAGE_SIZE;
1062 vmsplice_ret = vmsplice(pipe_fds[1], &iov, 1, SPLICE_F_GIFT);
1063 dprintf1("vmsplice() ret: %d\n", vmsplice_ret);
1064 pkey_assert(vmsplice_ret == -1);
1070 void test_kernel_gup_write_to_write_disabled_region(int *ptr, u16 pkey)
1072 int ignored = 0xdada;
1074 int some_int = __LINE__;
1076 dprintf1("disabling write to PKEY[%02d], "
1077 "doing futex gunk in buffer\n", pkey);
1079 pkey_write_deny(pkey);
1080 futex_ret = syscall(SYS_futex, ptr, FUTEX_WAIT, some_int-1, NULL,
1082 if (DEBUG_LEVEL > 0)
1084 dprintf1("futex() ret: %d\n", futex_ret);
1087 /* Assumes that all pkeys other than 'pkey' are unallocated */
1088 void test_pkey_syscalls_on_non_allocated_pkey(int *ptr, u16 pkey)
1093 /* Note: 0 is the default pkey, so don't mess with it */
1094 for (i = 1; i < NR_PKEYS; i++) {
1098 dprintf1("trying get/set/free to non-allocated pkey: %2d\n", i);
1099 err = sys_pkey_free(i);
1102 /* not enforced when pkey_get() is not a syscall
1103 err = pkey_get(i, 0);
1104 pkey_assert(err < 0);
1107 err = sys_pkey_free(i);
1110 err = sys_mprotect_pkey(ptr, PAGE_SIZE, PROT_READ, i);
1115 /* Assumes that all pkeys other than 'pkey' are unallocated */
1116 void test_pkey_syscalls_bad_args(int *ptr, u16 pkey)
1119 int bad_flag = (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE) + 1;
1120 int bad_pkey = NR_PKEYS+99;
1122 /* not enforced when pkey_get() is not a syscall
1123 err = pkey_get(bad_pkey, bad_flag);
1124 pkey_assert(err < 0);
1127 /* pass a known-invalid pkey in: */
1128 err = sys_mprotect_pkey(ptr, PAGE_SIZE, PROT_READ, bad_pkey);
1132 void become_child(void)
1137 pkey_assert(forkret >= 0);
1138 dprintf3("[%d] fork() ret: %d\n", getpid(), forkret);
1147 /* Assumes that all pkeys other than 'pkey' are unallocated */
1148 void test_pkey_alloc_exhaust(int *ptr, u16 pkey)
1150 unsigned long flags;
1151 unsigned long init_val;
1153 int allocated_pkeys[NR_PKEYS] = {0};
1154 int nr_allocated_pkeys = 0;
1157 for (i = 0; i < NR_PKEYS*3; i++) {
1159 dprintf1("%s() alloc loop: %d\n", __func__, i);
1160 new_pkey = alloc_pkey();
1161 dprintf4("%s()::%d, err: %d pkru: 0x%x shadow: 0x%x\n", __func__,
1162 __LINE__, err, __rdpkru(), shadow_pkru);
1163 rdpkru(); /* for shadow checking */
1164 dprintf2("%s() errno: %d ENOSPC: %d\n", __func__, errno, ENOSPC);
1165 if ((new_pkey == -1) && (errno == ENOSPC)) {
1166 dprintf2("%s() failed to allocate pkey after %d tries\n",
1167 __func__, nr_allocated_pkeys);
1170 * Ensure the number of successes never
1171 * exceeds the number of keys supported
1174 pkey_assert(nr_allocated_pkeys < NR_PKEYS);
1175 allocated_pkeys[nr_allocated_pkeys++] = new_pkey;
1179 * Make sure that allocation state is properly
1180 * preserved across fork().
1182 if (i == NR_PKEYS*2)
1186 dprintf3("%s()::%d\n", __func__, __LINE__);
1189 * There are 16 pkeys supported in hardware. One is taken
1190 * up for the default (0) and another can be taken up by
1191 * an execute-only mapping. Ensure that we can allocate
1192 * at least 14 (16-2).
1194 pkey_assert(i >= NR_PKEYS-2);
1196 for (i = 0; i < nr_allocated_pkeys; i++) {
1197 err = sys_pkey_free(allocated_pkeys[i]);
1199 rdpkru(); /* for shadow checking */
1203 void test_ptrace_of_child(int *ptr, u16 pkey)
1205 __attribute__((__unused__)) int peek_result;
1211 * This is the "control" for our little expermient. Make sure
1212 * we can always access it when ptracing.
1214 int *plain_ptr_unaligned = malloc(HPAGE_SIZE);
1215 int *plain_ptr = ALIGN_PTR_UP(plain_ptr_unaligned, PAGE_SIZE);
1218 * Fork a child which is an exact copy of this process, of course.
1219 * That means we can do all of our tests via ptrace() and then plain
1220 * memory access and ensure they work differently.
1222 child_pid = fork_lazy_child();
1223 dprintf1("[%d] child pid: %d\n", getpid(), child_pid);
1225 ret = ptrace(PTRACE_ATTACH, child_pid, ignored, ignored);
1228 dprintf1("[%d] attach ret: %ld %d\n", getpid(), ret, __LINE__);
1229 pkey_assert(ret != -1);
1230 ret = waitpid(child_pid, &status, WUNTRACED);
1231 if ((ret != child_pid) || !(WIFSTOPPED(status))) {
1232 fprintf(stderr, "weird waitpid result %ld stat %x\n",
1236 dprintf2("waitpid ret: %ld\n", ret);
1237 dprintf2("waitpid status: %d\n", status);
1239 pkey_access_deny(pkey);
1240 pkey_write_deny(pkey);
1242 /* Write access, untested for now:
1243 ret = ptrace(PTRACE_POKEDATA, child_pid, peek_at, data);
1244 pkey_assert(ret != -1);
1245 dprintf1("poke at %p: %ld\n", peek_at, ret);
1249 * Try to access the pkey-protected "ptr" via ptrace:
1251 ret = ptrace(PTRACE_PEEKDATA, child_pid, ptr, ignored);
1252 /* expect it to work, without an error: */
1253 pkey_assert(ret != -1);
1254 /* Now access from the current task, and expect an exception: */
1255 peek_result = read_ptr(ptr);
1256 expected_pk_fault(pkey);
1259 * Try to access the NON-pkey-protected "plain_ptr" via ptrace:
1261 ret = ptrace(PTRACE_PEEKDATA, child_pid, plain_ptr, ignored);
1262 /* expect it to work, without an error: */
1263 pkey_assert(ret != -1);
1264 /* Now access from the current task, and expect NO exception: */
1265 peek_result = read_ptr(plain_ptr);
1266 do_not_expect_pk_fault();
1268 ret = ptrace(PTRACE_DETACH, child_pid, ignored, 0);
1269 pkey_assert(ret != -1);
1271 ret = kill(child_pid, SIGKILL);
1272 pkey_assert(ret != -1);
1276 free(plain_ptr_unaligned);
1279 void test_executing_on_unreadable_memory(int *ptr, u16 pkey)
1286 p1 = ALIGN_PTR_UP(&lots_o_noops_around_write, PAGE_SIZE);
1287 dprintf3("&lots_o_noops: %p\n", &lots_o_noops_around_write);
1288 /* lots_o_noops_around_write should be page-aligned already */
1289 assert(p1 == &lots_o_noops_around_write);
1291 /* Point 'p1' at the *second* page of the function: */
1294 madvise(p1, PAGE_SIZE, MADV_DONTNEED);
1295 lots_o_noops_around_write(&scratch);
1296 ptr_contents = read_ptr(p1);
1297 dprintf2("ptr (%p) contents@%d: %x\n", p1, __LINE__, ptr_contents);
1299 ret = mprotect_pkey(p1, PAGE_SIZE, PROT_EXEC, (u64)pkey);
1301 pkey_access_deny(pkey);
1303 dprintf2("pkru: %x\n", rdpkru());
1306 * Make sure this is an *instruction* fault
1308 madvise(p1, PAGE_SIZE, MADV_DONTNEED);
1309 lots_o_noops_around_write(&scratch);
1310 do_not_expect_pk_fault();
1311 ptr_contents = read_ptr(p1);
1312 dprintf2("ptr (%p) contents@%d: %x\n", p1, __LINE__, ptr_contents);
1313 expected_pk_fault(pkey);
1316 void test_mprotect_pkey_on_unsupported_cpu(int *ptr, u16 pkey)
1318 int size = PAGE_SIZE;
1321 if (cpu_has_pku()) {
1322 dprintf1("SKIP: %s: no CPU support\n", __func__);
1326 sret = syscall(SYS_mprotect_key, ptr, size, PROT_READ, pkey);
1327 pkey_assert(sret < 0);
1330 void (*pkey_tests[])(int *ptr, u16 pkey) = {
1331 test_read_of_write_disabled_region,
1332 test_read_of_access_disabled_region,
1333 test_write_of_write_disabled_region,
1334 test_write_of_access_disabled_region,
1335 test_kernel_write_of_access_disabled_region,
1336 test_kernel_write_of_write_disabled_region,
1337 test_kernel_gup_of_access_disabled_region,
1338 test_kernel_gup_write_to_write_disabled_region,
1339 test_executing_on_unreadable_memory,
1340 test_ptrace_of_child,
1341 test_pkey_syscalls_on_non_allocated_pkey,
1342 test_pkey_syscalls_bad_args,
1343 test_pkey_alloc_exhaust,
1346 void run_tests_once(void)
1349 int prot = PROT_READ|PROT_WRITE;
1351 for (test_nr = 0; test_nr < ARRAY_SIZE(pkey_tests); test_nr++) {
1353 int orig_pkru_faults = pkru_faults;
1355 dprintf1("======================\n");
1356 dprintf1("test %d preparing...\n", test_nr);
1359 pkey = alloc_random_pkey();
1360 dprintf1("test %d starting with pkey: %d\n", test_nr, pkey);
1361 ptr = malloc_pkey(PAGE_SIZE, prot, pkey);
1362 dprintf1("test %d starting...\n", test_nr);
1363 pkey_tests[test_nr](ptr, pkey);
1364 dprintf1("freeing test memory: %p\n", ptr);
1365 free_pkey_malloc(ptr);
1366 sys_pkey_free(pkey);
1368 dprintf1("pkru_faults: %d\n", pkru_faults);
1369 dprintf1("orig_pkru_faults: %d\n", orig_pkru_faults);
1374 printf("test %2d PASSED (itertation %d)\n", test_nr, iteration_nr);
1375 dprintf1("======================\n\n");
1380 void pkey_setup_shadow(void)
1382 shadow_pkru = __rdpkru();
1387 int nr_iterations = 22;
1389 srand((unsigned int)time(NULL));
1393 printf("has pku: %d\n", cpu_has_pku());
1395 if (!cpu_has_pku()) {
1396 int size = PAGE_SIZE;
1399 printf("running PKEY tests for unsupported CPU/OS\n");
1401 ptr = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
1402 assert(ptr != (void *)-1);
1403 test_mprotect_pkey_on_unsupported_cpu(ptr, 1);
1407 pkey_setup_shadow();
1408 printf("startup pkru: %x\n", rdpkru());
1411 while (nr_iterations-- > 0)
1414 printf("done (all tests OK)\n");