2 * mpx-mini-test.c: routines to test Intel MPX (Memory Protection eXtentions)
5 * "Ren, Qiaowei" <qiaowei.ren@intel.com>
6 * "Wei, Gang" <gang.wei@intel.com>
7 * "Hansen, Dave" <dave.hansen@intel.com>
9 * This program is free software; you can redistribute it and/or modify it
10 * under the terms and conditions of the GNU General Public License,
15 * 2014-12-05: Dave Hansen: fixed all of the compiler warnings, and made sure
19 int inspect_every_this_many_mallocs = 100;
20 int zap_all_every_this_many_mallocs = 1000;
23 #define _LARGEFILE64_SOURCE
34 #include <sys/types.h>
40 #include "mpx-debug.h"
43 #ifndef __always_inline
44 #define __always_inline inline __attribute__((always_inline)
47 #ifndef TEST_DURATION_SECS
48 #define TEST_DURATION_SECS 3
51 void write_int_to(char *prefix, char *file, int int_to_write)
54 int fd = open(file, O_RDWR);
59 len = snprintf(buf, sizeof(buf), "%s%d", prefix, int_to_write);
61 assert(len < sizeof(buf));
62 ret = write(fd, buf, len);
68 void write_pid_to(char *prefix, char *file)
70 write_int_to(prefix, file, getpid());
75 /* tracing events dir */
76 #define TED "/sys/kernel/debug/tracing/events/"
78 write_pid_to("common_pid=", TED "signal/filter");
79 write_pid_to("common_pid=", TED "exceptions/filter");
80 write_int_to("", TED "signal/enable", 1);
81 write_int_to("", TED "exceptions/enable", 1);
83 write_pid_to("", "/sys/kernel/debug/tracing/set_ftrace_pid");
84 write_int_to("", "/sys/kernel/debug/tracing/trace", 0);
87 #define test_failed() __test_failed(__FILE__, __LINE__)
88 static void __test_failed(char *f, int l)
90 fprintf(stderr, "abort @ %s::%d\n", f, l);
95 #define eprintf(args...) fprintf(stderr, args)
99 /* i386 directory size is 4MB */
100 #define REG_IP_IDX REG_EIP
103 #define XSAVE_OFFSET_IN_FPMEM sizeof(struct _libc_fpstate)
106 * __cpuid() is from the Linux Kernel:
108 static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
109 unsigned int *ecx, unsigned int *edx)
111 /* ecx is often an input as well as an output. */
121 : "0" (*eax), "2" (*ecx));
126 #define REG_IP_IDX REG_RIP
127 #define REX_PREFIX "0x48, "
129 #define XSAVE_OFFSET_IN_FPMEM 0
132 * __cpuid() is from the Linux Kernel:
134 static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
135 unsigned int *ecx, unsigned int *edx)
137 /* ecx is often an input as well as an output. */
144 : "0" (*eax), "2" (*ecx));
147 #endif /* !__i386__ */
149 struct xsave_hdr_struct {
151 uint64_t reserved1[2];
152 uint64_t reserved2[5];
153 } __attribute__((packed));
155 struct bndregs_struct {
157 } __attribute__((packed));
159 struct bndcsr_struct {
162 } __attribute__((packed));
164 struct xsave_struct {
165 uint8_t fpu_sse[512];
166 struct xsave_hdr_struct xsave_hdr;
169 struct bndregs_struct bndregs;
170 struct bndcsr_struct bndcsr;
171 } __attribute__((packed));
173 uint8_t __attribute__((__aligned__(64))) buffer[4096];
174 struct xsave_struct *xsave_buf = (struct xsave_struct *)buffer;
176 uint8_t __attribute__((__aligned__(64))) test_buffer[4096];
177 struct xsave_struct *xsave_test_buf = (struct xsave_struct *)test_buffer;
179 uint64_t num_bnd_chk;
181 static __always_inline void xrstor_state(struct xsave_struct *fx, uint64_t mask)
183 uint32_t lmask = mask;
184 uint32_t hmask = mask >> 32;
186 asm volatile(".byte " REX_PREFIX "0x0f,0xae,0x2f\n\t"
187 : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
191 static __always_inline void xsave_state_1(void *_fx, uint64_t mask)
193 uint32_t lmask = mask;
194 uint32_t hmask = mask >> 32;
195 unsigned char *fx = _fx;
197 asm volatile(".byte " REX_PREFIX "0x0f,0xae,0x27\n\t"
198 : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
202 static inline uint64_t xgetbv(uint32_t index)
206 asm volatile(".byte 0x0f,0x01,0xd0" /* xgetbv */
207 : "=a" (eax), "=d" (edx)
209 return eax + ((uint64_t)edx << 32);
212 static uint64_t read_mpx_status_sig(ucontext_t *uctxt)
214 memset(buffer, 0, sizeof(buffer));
216 (uint8_t *)uctxt->uc_mcontext.fpregs + XSAVE_OFFSET_IN_FPMEM,
217 sizeof(struct xsave_struct));
219 return xsave_buf->bndcsr.status_reg;
224 static uint8_t *get_next_inst_ip(uint8_t *addr)
233 /* determine the prefix. */
242 /* look for rex prefix */
243 if ((*ip & 0x40) == 0x40)
246 /* Make sure we have a MPX instruction. */
250 /* Skip the op code byte. */
253 /* Get the modrm byte. */
256 /* Break it down into parts. */
260 /* Init the parts of the address mode. */
263 /* Is it a mem mode? */
265 /* look for scaled indexed addressing */
286 /* MODRM addressing */
289 /* DISP32 addressing, no base */
308 static inline void *__si_bounds_lower(siginfo_t *si)
313 static inline void *__si_bounds_upper(siginfo_t *si)
320 * This deals with old version of _sigfault in some distros:
344 static inline void **__si_bounds_hack(siginfo_t *si)
346 void *sigfault = &si->_sifields._sigfault;
347 void *end_sigfault = sigfault + sizeof(si->_sifields._sigfault);
348 int *trapno = (int*)end_sigfault;
349 /* skip _trapno and _addr_lsb */
350 void **__si_lower = (void**)(trapno + 2);
355 static inline void *__si_bounds_lower(siginfo_t *si)
357 return *__si_bounds_hack(si);
360 static inline void *__si_bounds_upper(siginfo_t *si)
362 return *(__si_bounds_hack(si) + 1);
367 static int expected_bnd_index = -1;
368 uint64_t shadow_plb[NR_MPX_BOUNDS_REGISTERS][2]; /* shadow MPX bound registers */
369 unsigned long shadow_map[NR_MPX_BOUNDS_REGISTERS];
372 * The kernel is supposed to provide some information about the bounds
373 * exception in the siginfo. It should match what we have in the bounds
374 * registers that we are checking against. Just check against the shadow copy
375 * since it is easily available, and we also check that *it* matches the real
378 void check_siginfo_vs_shadow(siginfo_t* si)
381 void *shadow_lower = (void *)(unsigned long)shadow_plb[expected_bnd_index][0];
382 void *shadow_upper = (void *)(unsigned long)shadow_plb[expected_bnd_index][1];
384 if ((expected_bnd_index < 0) ||
385 (expected_bnd_index >= NR_MPX_BOUNDS_REGISTERS)) {
386 fprintf(stderr, "ERROR: invalid expected_bnd_index: %d\n",
390 if (__si_bounds_lower(si) != shadow_lower)
392 if (__si_bounds_upper(si) != shadow_upper)
396 fprintf(stderr, "ERROR: siginfo bounds do not match "
397 "shadow bounds for register %d\n", expected_bnd_index);
402 void handler(int signum, siginfo_t *si, void *vucontext)
405 ucontext_t *uctxt = vucontext;
409 dprintf1("entered signal handler\n");
411 trapno = uctxt->uc_mcontext.gregs[REG_TRAPNO];
412 ip = uctxt->uc_mcontext.gregs[REG_IP_IDX];
415 typeof(si->si_addr) *si_addr_ptr = &si->si_addr;
416 uint64_t status = read_mpx_status_sig(uctxt);
417 uint64_t br_reason = status & 0x3;
420 dprintf1("#BR 0x%jx (total seen: %d)\n", status, br_count);
422 #define SEGV_BNDERR 3 /* failed address bound checks */
424 dprintf2("Saw a #BR! status 0x%jx at %016lx br_reason: %jx\n",
425 status, ip, br_reason);
426 dprintf2("si_signo: %d\n", si->si_signo);
427 dprintf2(" signum: %d\n", signum);
428 dprintf2("info->si_code == SEGV_BNDERR: %d\n",
429 (si->si_code == SEGV_BNDERR));
430 dprintf2("info->si_code: %d\n", si->si_code);
431 dprintf2("info->si_lower: %p\n", __si_bounds_lower(si));
432 dprintf2("info->si_upper: %p\n", __si_bounds_upper(si));
434 check_siginfo_vs_shadow(si);
436 for (i = 0; i < 8; i++)
437 dprintf3("[%d]: %p\n", i, si_addr_ptr[i]);
439 case 0: /* traditional BR */
441 "Undefined status with bound exception:%jx\n",
444 case 1: /* #BR MPX bounds exception */
445 /* these are normal and we expect to see them */
446 dprintf1("bounds exception (normal): status 0x%jx at %p si_addr: %p\n",
447 status, (void *)ip, si->si_addr);
449 uctxt->uc_mcontext.gregs[REG_IP_IDX] =
450 (greg_t)get_next_inst_ip((uint8_t *)ip);
453 fprintf(stderr, "#BR status == 2, missing bounds table,"
454 "kernel should have handled!!\n");
458 fprintf(stderr, "bound check error: status 0x%jx at %p\n",
461 uctxt->uc_mcontext.gregs[REG_IP_IDX] =
462 (greg_t)get_next_inst_ip((uint8_t *)ip);
463 fprintf(stderr, "bound check error: si_addr %p\n", si->si_addr);
466 } else if (trapno == 14) {
467 eprintf("ERROR: In signal handler, page fault, trapno = %d, ip = %016lx\n",
469 eprintf("si_addr %p\n", si->si_addr);
470 eprintf("REG_ERR: %lx\n", (unsigned long)uctxt->uc_mcontext.gregs[REG_ERR]);
473 eprintf("unexpected trap %d! at 0x%lx\n", trapno, ip);
474 eprintf("si_addr %p\n", si->si_addr);
475 eprintf("REG_ERR: %lx\n", (unsigned long)uctxt->uc_mcontext.gregs[REG_ERR]);
480 static inline void cpuid_count(unsigned int op, int count,
481 unsigned int *eax, unsigned int *ebx,
482 unsigned int *ecx, unsigned int *edx)
486 __cpuid(eax, ebx, ecx, edx);
489 #define XSTATE_CPUID 0x0000000d
492 * List of XSAVE features Linux knows about:
501 XSTATE_BIT_ZMM_Hi256,
507 #define XSTATE_FP (1 << XSTATE_BIT_FP)
508 #define XSTATE_SSE (1 << XSTATE_BIT_SSE)
509 #define XSTATE_YMM (1 << XSTATE_BIT_YMM)
510 #define XSTATE_BNDREGS (1 << XSTATE_BIT_BNDREGS)
511 #define XSTATE_BNDCSR (1 << XSTATE_BIT_BNDCSR)
512 #define XSTATE_OPMASK (1 << XSTATE_BIT_OPMASK)
513 #define XSTATE_ZMM_Hi256 (1 << XSTATE_BIT_ZMM_Hi256)
514 #define XSTATE_Hi16_ZMM (1 << XSTATE_BIT_Hi16_ZMM)
516 #define MPX_XSTATES (XSTATE_BNDREGS | XSTATE_BNDCSR) /* 0x18 */
518 bool one_bit(unsigned int x, int bit)
520 return !!(x & (1<<bit));
523 void print_state_component(int state_bit_nr, char *name)
525 unsigned int eax, ebx, ecx, edx;
526 unsigned int state_component_size;
527 unsigned int state_component_supervisor;
528 unsigned int state_component_user;
529 unsigned int state_component_aligned;
531 /* See SDM Section 13.2 */
532 cpuid_count(XSTATE_CPUID, state_bit_nr, &eax, &ebx, &ecx, &edx);
533 assert(eax || ebx || ecx);
534 state_component_size = eax;
535 state_component_supervisor = ((!ebx) && one_bit(ecx, 0));
536 state_component_user = !one_bit(ecx, 0);
537 state_component_aligned = one_bit(ecx, 1);
538 printf("%8s: size: %d user: %d supervisor: %d aligned: %d\n",
540 state_component_size, state_component_user,
541 state_component_supervisor, state_component_aligned);
545 /* Intel-defined CPU features, CPUID level 0x00000001 (ecx) */
546 #define XSAVE_FEATURE_BIT (26) /* XSAVE/XRSTOR/XSETBV/XGETBV */
547 #define OSXSAVE_FEATURE_BIT (27) /* XSAVE enabled in the OS */
549 bool check_mpx_support(void)
551 unsigned int eax, ebx, ecx, edx;
553 cpuid_count(1, 0, &eax, &ebx, &ecx, &edx);
555 /* We can't do much without XSAVE, so just make these assert()'s */
556 if (!one_bit(ecx, XSAVE_FEATURE_BIT)) {
557 fprintf(stderr, "processor lacks XSAVE, can not run MPX tests\n");
561 if (!one_bit(ecx, OSXSAVE_FEATURE_BIT)) {
562 fprintf(stderr, "processor lacks OSXSAVE, can not run MPX tests\n");
566 /* CPUs not supporting the XSTATE CPUID leaf do not support MPX */
567 /* Is this redundant with the feature bit checks? */
568 cpuid_count(0, 0, &eax, &ebx, &ecx, &edx);
569 if (eax < XSTATE_CPUID) {
570 fprintf(stderr, "processor lacks XSTATE CPUID leaf,"
571 " can not run MPX tests\n");
575 printf("XSAVE is supported by HW & OS\n");
577 cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
579 printf("XSAVE processor supported state mask: 0x%x\n", eax);
580 printf("XSAVE OS supported state mask: 0x%jx\n", xgetbv(0));
582 /* Make sure that the MPX states are enabled in in XCR0 */
583 if ((eax & MPX_XSTATES) != MPX_XSTATES) {
584 fprintf(stderr, "processor lacks MPX XSTATE(s), can not run MPX tests\n");
588 /* Make sure the MPX states are supported by XSAVE* */
589 if ((xgetbv(0) & MPX_XSTATES) != MPX_XSTATES) {
590 fprintf(stderr, "MPX XSTATE(s) no enabled in XCR0, "
591 "can not run MPX tests\n");
595 print_state_component(XSTATE_BIT_BNDREGS, "BNDREGS");
596 print_state_component(XSTATE_BIT_BNDCSR, "BNDCSR");
601 void enable_mpx(void *l1base)
603 /* enable point lookup */
604 memset(buffer, 0, sizeof(buffer));
605 xrstor_state(xsave_buf, 0x18);
607 xsave_buf->xsave_hdr.xstate_bv = 0x10;
608 xsave_buf->bndcsr.cfg_reg_u = (unsigned long)l1base | 1;
609 xsave_buf->bndcsr.status_reg = 0;
611 dprintf2("bf xrstor\n");
612 dprintf2("xsave cndcsr: status %jx, configu %jx\n",
613 xsave_buf->bndcsr.status_reg, xsave_buf->bndcsr.cfg_reg_u);
614 xrstor_state(xsave_buf, 0x18);
615 dprintf2("after xrstor\n");
617 xsave_state_1(xsave_buf, 0x18);
619 dprintf1("xsave bndcsr: status %jx, configu %jx\n",
620 xsave_buf->bndcsr.status_reg, xsave_buf->bndcsr.cfg_reg_u);
623 #include <sys/prctl.h>
625 struct mpx_bounds_dir *bounds_dir_ptr;
627 unsigned long __bd_incore(const char *func, int line)
629 unsigned long ret = nr_incore(bounds_dir_ptr, MPX_BOUNDS_DIR_SIZE_BYTES);
632 #define bd_incore() __bd_incore(__func__, __LINE__)
634 void check_clear(void *ptr, unsigned long sz)
638 for (i = ptr; (void *)i < ptr + sz; i++) {
640 dprintf1("%p is NOT clear at %p\n", ptr, i);
644 dprintf1("%p is clear for %lx\n", ptr, sz);
647 void check_clear_bd(void)
649 check_clear(bounds_dir_ptr, 2UL << 30);
652 #define USE_MALLOC_FOR_BOUNDS_DIR 1
653 bool process_specific_init(void)
657 /* Guarantee we have the space to align it, add padding: */
658 unsigned long pad = getpagesize();
660 size = 2UL << 30; /* 2GB */
661 if (sizeof(unsigned long) == 4)
662 size = 4UL << 20; /* 4MB */
663 dprintf1("trying to allocate %ld MB bounds directory\n", (size >> 20));
665 if (USE_MALLOC_FOR_BOUNDS_DIR) {
668 dir = malloc(size + pad);
670 _dir = (unsigned long)dir;
676 * This makes debugging easier because the address
677 * calculations are simpler:
679 dir = mmap((void *)0x200000000000, size + pad,
680 PROT_READ|PROT_WRITE,
681 MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
682 if (dir == (void *)-1) {
683 perror("unable to allocate bounds directory");
686 check_clear(dir, size);
688 bounds_dir_ptr = (void *)dir;
689 madvise(bounds_dir_ptr, size, MADV_NOHUGEPAGE);
691 dprintf1("bounds directory: 0x%p -> 0x%p\n", bounds_dir_ptr,
692 (char *)bounds_dir_ptr + size);
693 check_clear(dir, size);
695 check_clear(dir, size);
696 if (prctl(43, 0, 0, 0, 0)) {
697 printf("no MPX support\n");
704 bool process_specific_finish(void)
707 printf("no MPX support\n");
716 struct sigaction newact;
717 struct sigaction oldact;
719 /* #BR is mapped to sigsegv */
720 int signum = SIGSEGV;
722 newact.sa_handler = 0; /* void(*)(int)*/
723 newact.sa_sigaction = handler; /* void (*)(int, siginfo_t*, void *) */
725 /*sigset_t - signals to block while in the handler */
726 /* get the old signal mask. */
727 rs = sigprocmask(SIG_SETMASK, 0, &newact.sa_mask);
730 /* call sa_sigaction, not sa_handler*/
731 newact.sa_flags = SA_SIGINFO;
733 newact.sa_restorer = 0; /* void(*)(), obsolete */
734 r = sigaction(signum, &newact, &oldact);
738 void mpx_prepare(void)
740 dprintf2("%s()\n", __func__);
742 process_specific_init();
745 void mpx_cleanup(void)
747 printf("%s(): %jd BRs. bye...\n", __func__, num_bnd_chk);
748 process_specific_finish();
751 /*-------------- the following is test case ---------------*/
758 uint64_t num_lower_brs;
759 uint64_t num_upper_brs;
761 #define MPX_CONFIG_OFFSET 1024
762 #define MPX_BOUNDS_OFFSET 960
763 #define MPX_HEADER_OFFSET 512
764 #define MAX_ADDR_TESTED (1<<28)
765 #define TEST_ROUNDS 100
769 0F 1B /r BNDSTX-Store Extended Bounds Using Address Translation
770 66 0F 1A /r BNDMOV bnd1, bnd2/m128
771 66 0F 1B /r BNDMOV bnd1/m128, bnd2
772 F2 0F 1A /r BNDCU bnd, r/m64
773 F2 0F 1B /r BNDCN bnd, r/m64
774 F3 0F 1A /r BNDCL bnd, r/m64
775 F3 0F 1B /r BNDMK bnd, m64
778 static __always_inline void xsave_state(void *_fx, uint64_t mask)
780 uint32_t lmask = mask;
781 uint32_t hmask = mask >> 32;
782 unsigned char *fx = _fx;
784 asm volatile(".byte " REX_PREFIX "0x0f,0xae,0x27\n\t"
785 : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
789 static __always_inline void mpx_clear_bnd0(void)
793 /* F3 0F 1B /r BNDMK bnd, m64 */
794 /* f3 0f 1b 04 11 bndmk (%rcx,%rdx,1),%bnd0 */
795 asm volatile(".byte 0xf3,0x0f,0x1b,0x04,0x11\n\t"
796 : : "c" (ptr), "d" (size-1)
800 static __always_inline void mpx_make_bound_helper(unsigned long ptr,
803 /* F3 0F 1B /r BNDMK bnd, m64 */
804 /* f3 0f 1b 04 11 bndmk (%rcx,%rdx,1),%bnd0 */
805 asm volatile(".byte 0xf3,0x0f,0x1b,0x04,0x11\n\t"
806 : : "c" (ptr), "d" (size-1)
810 static __always_inline void mpx_check_lowerbound_helper(unsigned long ptr)
812 /* F3 0F 1A /r NDCL bnd, r/m64 */
813 /* f3 0f 1a 01 bndcl (%rcx),%bnd0 */
814 asm volatile(".byte 0xf3,0x0f,0x1a,0x01\n\t"
819 static __always_inline void mpx_check_upperbound_helper(unsigned long ptr)
821 /* F2 0F 1A /r BNDCU bnd, r/m64 */
822 /* f2 0f 1a 01 bndcu (%rcx),%bnd0 */
823 asm volatile(".byte 0xf2,0x0f,0x1a,0x01\n\t"
828 static __always_inline void mpx_movbndreg_helper()
830 /* 66 0F 1B /r BNDMOV bnd1/m128, bnd2 */
831 /* 66 0f 1b c2 bndmov %bnd0,%bnd2 */
833 asm volatile(".byte 0x66,0x0f,0x1b,0xc2\n\t");
836 static __always_inline void mpx_movbnd2mem_helper(uint8_t *mem)
838 /* 66 0F 1B /r BNDMOV bnd1/m128, bnd2 */
839 /* 66 0f 1b 01 bndmov %bnd0,(%rcx) */
840 asm volatile(".byte 0x66,0x0f,0x1b,0x01\n\t"
845 static __always_inline void mpx_movbnd_from_mem_helper(uint8_t *mem)
847 /* 66 0F 1A /r BNDMOV bnd1, bnd2/m128 */
848 /* 66 0f 1a 01 bndmov (%rcx),%bnd0 */
849 asm volatile(".byte 0x66,0x0f,0x1a,0x01\n\t"
854 static __always_inline void mpx_store_dsc_helper(unsigned long ptr_addr,
855 unsigned long ptr_val)
857 /* 0F 1B /r BNDSTX-Store Extended Bounds Using Address Translation */
858 /* 0f 1b 04 11 bndstx %bnd0,(%rcx,%rdx,1) */
859 asm volatile(".byte 0x0f,0x1b,0x04,0x11\n\t"
860 : : "c" (ptr_addr), "d" (ptr_val)
864 static __always_inline void mpx_load_dsc_helper(unsigned long ptr_addr,
865 unsigned long ptr_val)
867 /* 0F 1A /r BNDLDX-Load */
868 /*/ 0f 1a 04 11 bndldx (%rcx,%rdx,1),%bnd0 */
869 asm volatile(".byte 0x0f,0x1a,0x04,0x11\n\t"
870 : : "c" (ptr_addr), "d" (ptr_val)
874 void __print_context(void *__print_xsave_buffer, int line)
876 uint64_t *bounds = (uint64_t *)(__print_xsave_buffer + MPX_BOUNDS_OFFSET);
877 uint64_t *cfg = (uint64_t *)(__print_xsave_buffer + MPX_CONFIG_OFFSET);
880 eprintf("%s()::%d\n", "print_context", line);
881 for (i = 0; i < 4; i++) {
882 eprintf("bound[%d]: 0x%016lx 0x%016lx(0x%016lx)\n", i,
883 (unsigned long)bounds[i*2],
884 ~(unsigned long)bounds[i*2+1],
885 (unsigned long)bounds[i*2+1]);
888 eprintf("cpcfg: %jx cpstatus: %jx\n", cfg[0], cfg[1]);
890 #define print_context(x) __print_context(x, __LINE__)
892 #define dprint_context(x) print_context(x)
894 #define dprint_context(x) do{}while(0)
901 srand((unsigned int)time(NULL));
903 for (i = 0; i < 4; i++) {
904 shadow_plb[i][0] = 0;
905 shadow_plb[i][1] = ~(unsigned long)0;
909 long int __mpx_random(int line)
912 static long fake = 722122311;
919 #define mpx_random() __mpx_random(__LINE__)
921 uint8_t *get_random_addr()
923 uint8_t*addr = (uint8_t *)(unsigned long)(rand() % MAX_ADDR_TESTED);
924 return (addr - (unsigned long)addr % sizeof(uint8_t *));
927 static inline bool compare_context(void *__xsave_buffer)
929 uint64_t *bounds = (uint64_t *)(__xsave_buffer + MPX_BOUNDS_OFFSET);
932 for (i = 0; i < 4; i++) {
933 dprintf3("shadow[%d]{%016lx/%016lx}\nbounds[%d]{%016lx/%016lx}\n",
934 i, (unsigned long)shadow_plb[i][0], (unsigned long)shadow_plb[i][1],
935 i, (unsigned long)bounds[i*2], ~(unsigned long)bounds[i*2+1]);
936 if ((shadow_plb[i][0] != bounds[i*2]) ||
937 (shadow_plb[i][1] != ~(unsigned long)bounds[i*2+1])) {
938 eprintf("ERROR comparing shadow to real bound register %d\n", i);
939 eprintf("shadow{0x%016lx/0x%016lx}\nbounds{0x%016lx/0x%016lx}\n",
940 (unsigned long)shadow_plb[i][0], (unsigned long)shadow_plb[i][1],
941 (unsigned long)bounds[i*2], (unsigned long)bounds[i*2+1]);
949 void mkbnd_shadow(uint8_t *ptr, int index, long offset)
951 uint64_t *lower = (uint64_t *)&(shadow_plb[index][0]);
952 uint64_t *upper = (uint64_t *)&(shadow_plb[index][1]);
953 *lower = (unsigned long)ptr;
954 *upper = (unsigned long)ptr + offset - 1;
957 void check_lowerbound_shadow(uint8_t *ptr, int index)
959 uint64_t *lower = (uint64_t *)&(shadow_plb[index][0]);
960 if (*lower > (uint64_t)(unsigned long)ptr)
963 dprintf1("LowerBoundChk passed:%p\n", ptr);
966 void check_upperbound_shadow(uint8_t *ptr, int index)
968 uint64_t upper = *(uint64_t *)&(shadow_plb[index][1]);
969 if (upper < (uint64_t)(unsigned long)ptr)
972 dprintf1("UpperBoundChk passed:%p\n", ptr);
975 __always_inline void movbndreg_shadow(int src, int dest)
977 shadow_plb[dest][0] = shadow_plb[src][0];
978 shadow_plb[dest][1] = shadow_plb[src][1];
981 __always_inline void movbnd2mem_shadow(int src, unsigned long *dest)
983 unsigned long *lower = (unsigned long *)&(shadow_plb[src][0]);
984 unsigned long *upper = (unsigned long *)&(shadow_plb[src][1]);
989 __always_inline void movbnd_from_mem_shadow(unsigned long *src, int dest)
991 unsigned long *lower = (unsigned long *)&(shadow_plb[dest][0]);
992 unsigned long *upper = (unsigned long *)&(shadow_plb[dest][1]);
997 __always_inline void stdsc_shadow(int index, uint8_t *ptr, uint8_t *ptr_val)
999 shadow_map[0] = (unsigned long)shadow_plb[index][0];
1000 shadow_map[1] = (unsigned long)shadow_plb[index][1];
1001 shadow_map[2] = (unsigned long)ptr_val;
1002 dprintf3("%s(%d, %p, %p) set shadow map[2]: %p\n", __func__,
1003 index, ptr, ptr_val, ptr_val);
1007 void lddsc_shadow(int index, uint8_t *ptr, uint8_t *ptr_val)
1009 uint64_t lower = shadow_map[0];
1010 uint64_t upper = shadow_map[1];
1011 uint8_t *value = (uint8_t *)shadow_map[2];
1013 if (value != ptr_val) {
1014 dprintf2("%s(%d, %p, %p) init shadow bounds[%d] "
1015 "because %p != %p\n", __func__, index, ptr,
1016 ptr_val, index, value, ptr_val);
1017 shadow_plb[index][0] = 0;
1018 shadow_plb[index][1] = ~(unsigned long)0;
1020 shadow_plb[index][0] = lower;
1021 shadow_plb[index][1] = upper;
1026 static __always_inline void mpx_test_helper0(uint8_t *buf, uint8_t *ptr)
1028 mpx_make_bound_helper((unsigned long)ptr, 0x1800);
1031 static __always_inline void mpx_test_helper0_shadow(uint8_t *buf, uint8_t *ptr)
1033 mkbnd_shadow(ptr, 0, 0x1800);
1036 static __always_inline void mpx_test_helper1(uint8_t *buf, uint8_t *ptr)
1038 /* these are hard-coded to check bnd0 */
1039 expected_bnd_index = 0;
1040 mpx_check_lowerbound_helper((unsigned long)(ptr-1));
1041 mpx_check_upperbound_helper((unsigned long)(ptr+0x1800));
1042 /* reset this since we do not expect any more bounds exceptions */
1043 expected_bnd_index = -1;
1046 static __always_inline void mpx_test_helper1_shadow(uint8_t *buf, uint8_t *ptr)
1048 check_lowerbound_shadow(ptr-1, 0);
1049 check_upperbound_shadow(ptr+0x1800, 0);
1052 static __always_inline void mpx_test_helper2(uint8_t *buf, uint8_t *ptr)
1054 mpx_make_bound_helper((unsigned long)ptr, 0x1800);
1055 mpx_movbndreg_helper();
1056 mpx_movbnd2mem_helper(buf);
1057 mpx_make_bound_helper((unsigned long)(ptr+0x12), 0x1800);
1060 static __always_inline void mpx_test_helper2_shadow(uint8_t *buf, uint8_t *ptr)
1062 mkbnd_shadow(ptr, 0, 0x1800);
1063 movbndreg_shadow(0, 2);
1064 movbnd2mem_shadow(0, (unsigned long *)buf);
1065 mkbnd_shadow(ptr+0x12, 0, 0x1800);
1068 static __always_inline void mpx_test_helper3(uint8_t *buf, uint8_t *ptr)
1070 mpx_movbnd_from_mem_helper(buf);
1073 static __always_inline void mpx_test_helper3_shadow(uint8_t *buf, uint8_t *ptr)
1075 movbnd_from_mem_shadow((unsigned long *)buf, 0);
1078 static __always_inline void mpx_test_helper4(uint8_t *buf, uint8_t *ptr)
1080 mpx_store_dsc_helper((unsigned long)buf, (unsigned long)ptr);
1081 mpx_make_bound_helper((unsigned long)(ptr+0x12), 0x1800);
1084 static __always_inline void mpx_test_helper4_shadow(uint8_t *buf, uint8_t *ptr)
1086 stdsc_shadow(0, buf, ptr);
1087 mkbnd_shadow(ptr+0x12, 0, 0x1800);
1090 static __always_inline void mpx_test_helper5(uint8_t *buf, uint8_t *ptr)
1092 mpx_load_dsc_helper((unsigned long)buf, (unsigned long)ptr);
1095 static __always_inline void mpx_test_helper5_shadow(uint8_t *buf, uint8_t *ptr)
1097 lddsc_shadow(0, buf, ptr);
1100 #define NR_MPX_TEST_FUNCTIONS 6
1103 * For compatibility reasons, MPX will clear the bounds registers
1104 * when you make function calls (among other things). We have to
1105 * preserve the registers in between calls to the "helpers" since
1106 * they build on each other.
1108 * Be very careful not to make any function calls inside the
1109 * helpers, or anywhere else beween the xrstor and xsave.
1111 #define run_helper(helper_nr, buf, buf_shadow, ptr) do { \
1112 xrstor_state(xsave_test_buf, flags); \
1113 mpx_test_helper##helper_nr(buf, ptr); \
1114 xsave_state(xsave_test_buf, flags); \
1115 mpx_test_helper##helper_nr##_shadow(buf_shadow, ptr); \
1118 static void run_helpers(int nr, uint8_t *buf, uint8_t *buf_shadow, uint8_t *ptr)
1120 uint64_t flags = 0x18;
1122 dprint_context(xsave_test_buf);
1125 run_helper(0, buf, buf_shadow, ptr);
1128 run_helper(1, buf, buf_shadow, ptr);
1131 run_helper(2, buf, buf_shadow, ptr);
1134 run_helper(3, buf, buf_shadow, ptr);
1137 run_helper(4, buf, buf_shadow, ptr);
1140 run_helper(5, buf, buf_shadow, ptr);
1146 dprint_context(xsave_test_buf);
1149 unsigned long buf_shadow[1024]; /* used to check load / store descriptors */
1150 extern long inspect_me(struct mpx_bounds_dir *bounds_dir);
1152 long cover_buf_with_bt_entries(void *buf, long buf_len)
1157 unsigned long buf_len_in_ptrs;
1159 /* Fill about 1/100 of the space with bt entries */
1160 nr_to_fill = buf_len / (sizeof(unsigned long) * ratio);
1163 dprintf3("%s() nr_to_fill: %ld\n", __func__, nr_to_fill);
1165 /* Align the buffer to pointer size */
1166 while (((unsigned long)buf) % sizeof(void *)) {
1170 /* We are storing pointers, so make */
1171 buf_len_in_ptrs = buf_len / sizeof(void *);
1173 for (i = 0; i < nr_to_fill; i++) {
1174 long index = (mpx_random() % buf_len_in_ptrs);
1175 void *ptr = buf + index * sizeof(unsigned long);
1176 unsigned long ptr_addr = (unsigned long)ptr;
1178 /* ptr and size can be anything */
1179 mpx_make_bound_helper((unsigned long)ptr, 8);
1182 * take bnd0 and put it in to bounds tables "buf + index" is an
1183 * address inside the buffer where we are pretending that we
1184 * are going to put a pointer We do not, though because we will
1185 * never load entries from the table, so it doesn't matter.
1187 mpx_store_dsc_helper(ptr_addr, (unsigned long)ptr);
1188 dprintf4("storing bound table entry for %lx (buf start @ %p)\n",
1194 unsigned long align_down(unsigned long alignme, unsigned long align_to)
1196 return alignme & ~(align_to-1);
1199 unsigned long align_up(unsigned long alignme, unsigned long align_to)
1201 return (alignme + align_to - 1) & ~(align_to-1);
1205 * Using 1MB alignment guarantees that each no allocation
1206 * will overlap with another's bounds tables.
1208 * We have to cook our own allocator here. malloc() can
1209 * mix other allocation with ours which means that even
1210 * if we free all of our allocations, there might still
1211 * be bounds tables for the *areas* since there is other
1212 * valid memory there.
1214 * We also can't use malloc() because a free() of an area
1215 * might not free it back to the kernel. We want it
1216 * completely unmapped an malloc() does not guarantee
1220 long alignment = 4096;
1221 long sz_alignment = 4096;
1223 long alignment = 1 * MB;
1224 long sz_alignment = 1 * MB;
1226 void *mpx_mini_alloc(unsigned long sz)
1228 unsigned long long tries = 0;
1233 sz = align_up(sz, sz_alignment);
1235 try_at = last + alignment;
1237 ptr = mmap(try_at, sz, PROT_READ|PROT_WRITE,
1238 MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
1239 if (ptr == (void *)-1)
1245 try_at += alignment;
1248 * This isn't quite correct for 32-bit binaries
1249 * on 64-bit kernels since they can use the
1250 * entire 32-bit address space, but it's close
1253 if (try_at > (void *)0xC0000000)
1255 if (try_at > (void *)0x0000800000000000)
1257 try_at = (void *)0x0;
1258 if (!(++tries % 10000))
1259 dprintf1("stuck in %s(), tries: %lld\n", __func__, tries);
1263 dprintf3("mpx_mini_alloc(0x%lx) returning: %p\n", sz, ptr);
1266 void mpx_mini_free(void *ptr, long sz)
1268 dprintf2("%s() ptr: %p\n", __func__, ptr);
1269 if ((unsigned long)ptr > 0x100000000000) {
1270 dprintf1("uh oh !!!!!!!!!!!!!!! pointer too high: %p\n", ptr);
1273 sz = align_up(sz, sz_alignment);
1274 dprintf3("%s() ptr: %p before munmap\n", __func__, ptr);
1276 dprintf3("%s() ptr: %p DONE\n", __func__, ptr);
1279 #define NR_MALLOCS 100
1285 struct one_malloc mallocs[NR_MALLOCS];
1287 void free_one_malloc(int index)
1289 unsigned long free_ptr;
1292 if (!mallocs[index].ptr)
1295 mpx_mini_free(mallocs[index].ptr, mallocs[index].size);
1296 dprintf4("freed[%d]: %p\n", index, mallocs[index].ptr);
1298 free_ptr = (unsigned long)mallocs[index].ptr;
1300 dprintf4("lowerbits: %lx / %lx mask: %lx\n", free_ptr,
1301 (free_ptr & mask), mask);
1302 assert((free_ptr & mask) == 0);
1304 mallocs[index].ptr = NULL;
1308 #define MPX_BOUNDS_TABLE_COVERS 4096
1310 #define MPX_BOUNDS_TABLE_COVERS (1 * MB)
1312 void zap_everything(void)
1318 before_zap = inspect_me(bounds_dir_ptr);
1319 dprintf1("zapping everything start: %ld\n", before_zap);
1320 for (i = 0; i < NR_MALLOCS; i++)
1323 after_zap = inspect_me(bounds_dir_ptr);
1324 dprintf1("zapping everything done: %ld\n", after_zap);
1326 * We only guarantee to empty the thing out if our allocations are
1327 * exactly aligned on the boundaries of a boudns table.
1329 if ((alignment >= MPX_BOUNDS_TABLE_COVERS) &&
1330 (sz_alignment >= MPX_BOUNDS_TABLE_COVERS)) {
1334 assert(after_zap == 0);
1338 void do_one_malloc(void)
1340 static int malloc_counter;
1342 int rand_index = (mpx_random() % NR_MALLOCS);
1343 void *ptr = mallocs[rand_index].ptr;
1345 dprintf3("%s() enter\n", __func__);
1348 dprintf3("freeing one malloc at index: %d\n", rand_index);
1349 free_one_malloc(rand_index);
1350 if (mpx_random() % (NR_MALLOCS*3) == 3) {
1352 dprintf3("zapping some more\n");
1353 for (i = rand_index; i < NR_MALLOCS; i++)
1356 if ((mpx_random() % zap_all_every_this_many_mallocs) == 4)
1361 sz = (1 + mpx_random() % 1000) * 1000;
1362 ptr = mpx_mini_alloc(sz);
1365 * If we are failing allocations, just assume we
1366 * are out of memory and zap everything.
1368 dprintf3("zapping everything because out of memory\n");
1373 dprintf3("malloc: %p size: 0x%lx\n", ptr, sz);
1374 mallocs[rand_index].nr_filled_btes = cover_buf_with_bt_entries(ptr, sz);
1375 mallocs[rand_index].ptr = ptr;
1376 mallocs[rand_index].size = sz;
1378 if ((++malloc_counter) % inspect_every_this_many_mallocs == 0)
1379 inspect_me(bounds_dir_ptr);
1382 void run_timed_test(void (*test_func)(void))
1386 static time_t last_print;
1393 if ((now - start) > TEST_DURATION_SECS)
1399 if ((now - last_print > 1) || done) {
1400 printf("iteration %ld complete, OK so far\n", iteration);
1406 void check_bounds_table_frees(void)
1408 printf("executing unmaptest\n");
1409 inspect_me(bounds_dir_ptr);
1410 run_timed_test(&do_one_malloc);
1411 printf("done with malloc() fun\n");
1414 void insn_test_failed(int test_nr, int test_round, void *buf,
1415 void *buf_shadow, void *ptr)
1417 print_context(xsave_test_buf);
1418 eprintf("ERROR: test %d round %d failed\n", test_nr, test_round);
1419 while (test_nr == 5) {
1420 struct mpx_bt_entry *bte;
1421 struct mpx_bounds_dir *bd = (void *)bounds_dir_ptr;
1422 struct mpx_bd_entry *bde = mpx_vaddr_to_bd_entry(buf, bd);
1424 printf(" bd: %p\n", bd);
1425 printf("&bde: %p\n", bde);
1426 printf("*bde: %lx\n", *(unsigned long *)bde);
1427 if (!bd_entry_valid(bde))
1430 bte = mpx_vaddr_to_bt_entry(buf, bd);
1431 printf(" te: %p\n", bte);
1432 printf("bte[0]: %lx\n", bte->contents[0]);
1433 printf("bte[1]: %lx\n", bte->contents[1]);
1434 printf("bte[2]: %lx\n", bte->contents[2]);
1435 printf("bte[3]: %lx\n", bte->contents[3]);
1441 void check_mpx_insns_and_tables(void)
1445 int buf_size = (1024*1024);
1446 unsigned long *buf = malloc(buf_size);
1447 const int total_nr_tests = NR_MPX_TEST_FUNCTIONS * TEST_ROUNDS;
1450 memset(buf, 0, buf_size);
1451 memset(buf_shadow, 0, sizeof(buf_shadow));
1453 for (i = 0; i < TEST_ROUNDS; i++) {
1454 uint8_t *ptr = get_random_addr() + 8;
1456 for (j = 0; j < NR_MPX_TEST_FUNCTIONS; j++) {
1461 dprintf2("starting test %d round %d\n", j, i);
1462 dprint_context(xsave_test_buf);
1464 * test5 loads an address from the bounds tables.
1465 * The load will only complete if 'ptr' matches
1466 * the load and the store, so with random addrs,
1467 * the odds of this are very small. Make it
1468 * higher by only moving 'ptr' 1/10 times.
1470 if (random() % 10 <= 0)
1471 ptr = get_random_addr() + 8;
1472 dprintf3("random ptr{%p}\n", ptr);
1473 dprint_context(xsave_test_buf);
1474 run_helpers(j, (void *)buf, (void *)buf_shadow, ptr);
1475 dprint_context(xsave_test_buf);
1476 if (!compare_context(xsave_test_buf)) {
1477 insn_test_failed(j, i, buf, buf_shadow, ptr);
1482 dprint_context(xsave_test_buf);
1483 dprintf2("finished test %d round %d\n", j, i);
1485 dprint_context(xsave_test_buf);
1490 dprintf2("\nabout to free:\n");
1492 dprintf1("successes: %d\n", successes);
1493 dprintf1(" failures: %d\n", failures);
1494 dprintf1(" tests: %d\n", total_nr_tests);
1495 dprintf1(" expected: %jd #BRs\n", num_upper_brs + num_lower_brs);
1496 dprintf1(" saw: %d #BRs\n", br_count);
1498 eprintf("ERROR: non-zero number of failures\n");
1501 if (successes != total_nr_tests) {
1502 eprintf("ERROR: succeded fewer than number of tries (%d != %d)\n",
1503 successes, total_nr_tests);
1506 if (num_upper_brs + num_lower_brs != br_count) {
1507 eprintf("ERROR: unexpected number of #BRs: %jd %jd %d\n",
1508 num_upper_brs, num_lower_brs, br_count);
1509 eprintf("successes: %d\n", successes);
1510 eprintf(" failures: %d\n", failures);
1511 eprintf(" tests: %d\n", total_nr_tests);
1512 eprintf(" expected: %jd #BRs\n", num_upper_brs + num_lower_brs);
1513 eprintf(" saw: %d #BRs\n", br_count);
1519 * This is supposed to SIGSEGV nicely once the kernel
1520 * can no longer allocate vaddr space.
1522 void exhaust_vaddr_space(void)
1525 /* Try to make sure there is no room for a bounds table anywhere */
1526 unsigned long skip = MPX_BOUNDS_TABLE_SIZE_BYTES - PAGE_SIZE;
1528 unsigned long max_vaddr = 0xf7788000UL;
1530 unsigned long max_vaddr = 0x800000000000UL;
1533 dprintf1("%s() start\n", __func__);
1534 /* do not start at 0, we aren't allowed to map there */
1535 for (ptr = PAGE_SIZE; ptr < max_vaddr; ptr += skip) {
1537 int ret = madvise((void *)ptr, PAGE_SIZE, MADV_NORMAL);
1540 dprintf1("madvise() %lx ret: %d\n", ptr, ret);
1543 ptr_ret = mmap((void *)ptr, PAGE_SIZE, PROT_READ|PROT_WRITE,
1544 MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
1545 if (ptr_ret != (void *)ptr) {
1547 dprintf1("mmap(%lx) ret: %p\n", ptr, ptr_ret);
1550 if (!(ptr & 0xffffff))
1551 dprintf1("mmap(%lx) ret: %p\n", ptr, ptr_ret);
1553 for (ptr = PAGE_SIZE; ptr < max_vaddr; ptr += skip) {
1554 dprintf2("covering 0x%lx with bounds table entries\n", ptr);
1555 cover_buf_with_bt_entries((void *)ptr, PAGE_SIZE);
1557 dprintf1("%s() end\n", __func__);
1558 printf("done with vaddr space fun\n");
1561 void mpx_table_test(void)
1563 printf("starting mpx bounds table test\n");
1564 run_timed_test(check_mpx_insns_and_tables);
1565 printf("done with mpx bounds table test\n");
1568 int main(int argc, char **argv)
1571 int vaddrexhaust = 0;
1575 check_mpx_support();
1585 xsave_state((void *)xsave_test_buf, 0x1f);
1586 if (!compare_context(xsave_test_buf))
1587 printf("Init failed\n");
1589 for (i = 1; i < argc; i++) {
1590 if (!strcmp(argv[i], "unmaptest"))
1592 if (!strcmp(argv[i], "vaddrexhaust"))
1594 if (!strcmp(argv[i], "tabletest"))
1597 if (!(unmaptest || vaddrexhaust || tabletest)) {
1599 /* vaddrexhaust = 1; */
1603 check_bounds_table_frees();
1607 exhaust_vaddr_space();
1608 printf("%s completed successfully\n", argv[0]);
1612 #include "mpx-dig.c"