arch/x86/kernel/kprobes/opt.c

   1 /*
   2  *  Kernel Probes Jump Optimization (Optprobes)
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17  *
  18  * Copyright (C) IBM Corporation, 2002, 2004
  19  * Copyright (C) Hitachi Ltd., 2012
  20  */
  21 #include <linux/kprobes.h>
  22 #include <linux/ptrace.h>
  23 #include <linux/string.h>
  24 #include <linux/slab.h>
  25 #include <linux/hardirq.h>
  26 #include <linux/preempt.h>
  27 #include <linux/extable.h>
  28 #include <linux/kdebug.h>
  29 #include <linux/kallsyms.h>
  30 #include <linux/ftrace.h>
  31 #include <linux/frame.h>
  32
  33 #include <asm/text-patching.h>
  34 #include <asm/cacheflush.h>
  35 #include <asm/desc.h>
  36 #include <asm/pgtable.h>
  37 #include <asm/uaccess.h>
  38 #include <asm/alternative.h>
  39 #include <asm/insn.h>
  40 #include <asm/debugreg.h>
  41 #include <asm/nospec-branch.h>
  42 #include <asm/sections.h>
  43
  44 #include "common.h"
  45
  46 unsigned long __recover_optprobed_insn(kprobe_opcode_t *buf, unsigned long addr)
  47 {
  48         struct optimized_kprobe *op;
  49         struct kprobe *kp;
  50         long offs;
  51         int i;
  52
  53         for (i = 0; i < RELATIVEJUMP_SIZE; i++) {
  54                 kp = get_kprobe((void *)addr - i);
  55                 /* This function only handles jump-optimized kprobe */
  56                 if (kp && kprobe_optimized(kp)) {
  57                         op = container_of(kp, struct optimized_kprobe, kp);
  58                         /* If op->list is not empty, op is under optimizing */
  59                         if (list_empty(&op->list))
  60                                 goto found;
  61                 }
  62         }
  63
  64         return addr;
  65 found:
  66         /*
  67          * If the kprobe can be optimized, original bytes which can be
  68          * overwritten by jump destination address. In this case, original
  69          * bytes must be recovered from op->optinsn.copied_insn buffer.
  70          */
  71         memcpy(buf, (void *)addr, MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
  72         if (addr == (unsigned long)kp->addr) {
  73                 buf[0] = kp->opcode;
  74                 memcpy(buf + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE);
  75         } else {
  76                 offs = addr - (unsigned long)kp->addr - 1;
  77                 memcpy(buf, op->optinsn.copied_insn + offs, RELATIVE_ADDR_SIZE - offs);
  78         }
  79
  80         return (unsigned long)buf;
  81 }
  82
  83 /* Insert a move instruction which sets a pointer to eax/rdi (1st arg). */
  84 static void synthesize_set_arg1(kprobe_opcode_t *addr, unsigned long val)
  85 {
  86 #ifdef CONFIG_X86_64
  87         *addr++ = 0x48;
  88         *addr++ = 0xbf;
  89 #else
  90         *addr++ = 0xb8;
  91 #endif
  92         *(unsigned long *)addr = val;
  93 }
  94
  95 asm (
  96                         "optprobe_template_func:\n"
  97                         ".global optprobe_template_entry\n"
  98                         "optprobe_template_entry:\n"
  99 #ifdef CONFIG_X86_64
 100                         /* We don't bother saving the ss register */
 101                         "       pushq %rsp\n"
 102                         "       pushfq\n"
 103                         SAVE_REGS_STRING
 104                         "       movq %rsp, %rsi\n"
 105                         ".global optprobe_template_val\n"
 106                         "optprobe_template_val:\n"
 107                         ASM_NOP5
 108                         ASM_NOP5
 109                         ".global optprobe_template_call\n"
 110                         "optprobe_template_call:\n"
 111                         ASM_NOP5
 112                         /* Move flags to rsp */
 113                         "       movq 144(%rsp), %rdx\n"
 114                         "       movq %rdx, 152(%rsp)\n"
 115                         RESTORE_REGS_STRING
 116                         /* Skip flags entry */
 117                         "       addq $8, %rsp\n"
 118                         "       popfq\n"
 119 #else /* CONFIG_X86_32 */
 120                         "       pushf\n"
 121                         SAVE_REGS_STRING
 122                         "       movl %esp, %edx\n"
 123                         ".global optprobe_template_val\n"
 124                         "optprobe_template_val:\n"
 125                         ASM_NOP5
 126                         ".global optprobe_template_call\n"
 127                         "optprobe_template_call:\n"
 128                         ASM_NOP5
 129                         RESTORE_REGS_STRING
 130                         "       addl $4, %esp\n"        /* skip cs */
 131                         "       popf\n"
 132 #endif
 133                         ".global optprobe_template_end\n"
 134                         "optprobe_template_end:\n"
 135                         ".type optprobe_template_func, @function\n"
 136                         ".size optprobe_template_func, .-optprobe_template_func\n");
 137
 138 void optprobe_template_func(void);
 139 STACK_FRAME_NON_STANDARD(optprobe_template_func);
 140
 141 #define TMPL_MOVE_IDX \
 142         ((long)&optprobe_template_val - (long)&optprobe_template_entry)
 143 #define TMPL_CALL_IDX \
 144         ((long)&optprobe_template_call - (long)&optprobe_template_entry)
 145 #define TMPL_END_IDX \
 146         ((long)&optprobe_template_end - (long)&optprobe_template_entry)
 147
 148 #define INT3_SIZE sizeof(kprobe_opcode_t)
 149
 150 /* Optimized kprobe call back function: called from optinsn */
 151 static void
 152 optimized_callback(struct optimized_kprobe *op, struct pt_regs *regs)
 153 {
 154         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 155         unsigned long flags;
 156
 157         /* This is possible if op is under delayed unoptimizing */
 158         if (kprobe_disabled(&op->kp))
 159                 return;
 160
 161         local_irq_save(flags);
 162         if (kprobe_running()) {
 163                 kprobes_inc_nmissed_count(&op->kp);
 164         } else {
 165                 /* Save skipped registers */
 166 #ifdef CONFIG_X86_64
 167                 regs->cs = __KERNEL_CS;
 168 #else
 169                 regs->cs = __KERNEL_CS | get_kernel_rpl();
 170                 regs->gs = 0;
 171 #endif
 172                 regs->ip = (unsigned long)op->kp.addr + INT3_SIZE;
 173                 regs->orig_ax = ~0UL;
 174
 175                 __this_cpu_write(current_kprobe, &op->kp);
 176                 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
 177                 opt_pre_handler(&op->kp, regs);
 178                 __this_cpu_write(current_kprobe, NULL);
 179         }
 180         local_irq_restore(flags);
 181 }
 182 NOKPROBE_SYMBOL(optimized_callback);
 183
 184 static int copy_optimized_instructions(u8 *dest, u8 *src)
 185 {
 186         int len = 0, ret;
 187
 188         while (len < RELATIVEJUMP_SIZE) {
 189                 ret = __copy_instruction(dest + len, src + len);
 190                 if (!ret || !can_boost(dest + len, src + len))
 191                         return -EINVAL;
 192                 len += ret;
 193         }
 194         /* Check whether the address range is reserved */
 195         if (ftrace_text_reserved(src, src + len - 1) ||
 196             alternatives_text_reserved(src, src + len - 1) ||
 197             jump_label_text_reserved(src, src + len - 1))
 198                 return -EBUSY;
 199
 200         return len;
 201 }
 202
 203 /* Check whether insn is indirect jump */
 204 static int __insn_is_indirect_jump(struct insn *insn)
 205 {
 206         return ((insn->opcode.bytes[0] == 0xff &&
 207                 (X86_MODRM_REG(insn->modrm.value) & 6) == 4) || /* Jump */
 208                 insn->opcode.bytes[0] == 0xea); /* Segment based jump */
 209 }
 210
 211 /* Check whether insn jumps into specified address range */
 212 static int insn_jump_into_range(struct insn *insn, unsigned long start, int len)
 213 {
 214         unsigned long target = 0;
 215
 216         switch (insn->opcode.bytes[0]) {
 217         case 0xe0:      /* loopne */
 218         case 0xe1:      /* loope */
 219         case 0xe2:      /* loop */
 220         case 0xe3:      /* jcxz */
 221         case 0xe9:      /* near relative jump */
 222         case 0xeb:      /* short relative jump */
 223                 break;
 224         case 0x0f:
 225                 if ((insn->opcode.bytes[1] & 0xf0) == 0x80) /* jcc near */
 226                         break;
 227                 return 0;
 228         default:
 229                 if ((insn->opcode.bytes[0] & 0xf0) == 0x70) /* jcc short */
 230                         break;
 231                 return 0;
 232         }
 233         target = (unsigned long)insn->next_byte + insn->immediate.value;
 234
 235         return (start <= target && target <= start + len);
 236 }
 237
 238 static int insn_is_indirect_jump(struct insn *insn)
 239 {
 240         int ret = __insn_is_indirect_jump(insn);
 241
 242 #ifdef CONFIG_RETPOLINE
 243         /*
 244          * Jump to x86_indirect_thunk_* is treated as an indirect jump.
 245          * Note that even with CONFIG_RETPOLINE=y, the kernel compiled with
 246          * older gcc may use indirect jump. So we add this check instead of
 247          * replace indirect-jump check.
 248          */
 249         if (!ret)
 250                 ret = insn_jump_into_range(insn,
 251                                 (unsigned long)__indirect_thunk_start,
 252                                 (unsigned long)__indirect_thunk_end -
 253                                 (unsigned long)__indirect_thunk_start);
 254 #endif
 255         return ret;
 256 }
 257
 258 /* Decode whole function to ensure any instructions don't jump into target */
 259 static int can_optimize(unsigned long paddr)
 260 {
 261         unsigned long addr, size = 0, offset = 0;
 262         struct insn insn;
 263         kprobe_opcode_t buf[MAX_INSN_SIZE];
 264
 265         /* Lookup symbol including addr */
 266         if (!kallsyms_lookup_size_offset(paddr, &size, &offset))
 267                 return 0;
 268
 269         /*
 270          * Do not optimize in the entry code due to the unstable
 271          * stack handling.
 272          */
 273         if ((paddr >= (unsigned long)__entry_text_start) &&
 274             (paddr <  (unsigned long)__entry_text_end))
 275                 return 0;
 276
 277         /* Check there is enough space for a relative jump. */
 278         if (size - offset < RELATIVEJUMP_SIZE)
 279                 return 0;
 280
 281         /* Decode instructions */
 282         addr = paddr - offset;
 283         while (addr < paddr - offset + size) { /* Decode until function end */
 284                 unsigned long recovered_insn;
 285                 if (search_exception_tables(addr))
 286                         /*
 287                          * Since some fixup code will jumps into this function,
 288                          * we can't optimize kprobe in this function.
 289                          */
 290                         return 0;
 291                 recovered_insn = recover_probed_instruction(buf, addr);
 292                 if (!recovered_insn)
 293                         return 0;
 294                 kernel_insn_init(&insn, (void *)recovered_insn, MAX_INSN_SIZE);
 295                 insn_get_length(&insn);
 296                 /* Another subsystem puts a breakpoint */
 297                 if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION)
 298                         return 0;
 299                 /* Recover address */
 300                 insn.kaddr = (void *)addr;
 301                 insn.next_byte = (void *)(addr + insn.length);
 302                 /* Check any instructions don't jump into target */
 303                 if (insn_is_indirect_jump(&insn) ||
 304                     insn_jump_into_range(&insn, paddr + INT3_SIZE,
 305                                          RELATIVE_ADDR_SIZE))
 306                         return 0;
 307                 addr += insn.length;
 308         }
 309
 310         return 1;
 311 }
 312
 313 /* Check optimized_kprobe can actually be optimized. */
 314 int arch_check_optimized_kprobe(struct optimized_kprobe *op)
 315 {
 316         int i;
 317         struct kprobe *p;
 318
 319         for (i = 1; i < op->optinsn.size; i++) {
 320                 p = get_kprobe(op->kp.addr + i);
 321                 if (p && !kprobe_disabled(p))
 322                         return -EEXIST;
 323         }
 324
 325         return 0;
 326 }
 327
 328 /* Check the addr is within the optimized instructions. */
 329 int arch_within_optimized_kprobe(struct optimized_kprobe *op,
 330                                  unsigned long addr)
 331 {
 332         return ((unsigned long)op->kp.addr <= addr &&
 333                 (unsigned long)op->kp.addr + op->optinsn.size > addr);
 334 }
 335
 336 /* Free optimized instruction slot */
 337 static
 338 void __arch_remove_optimized_kprobe(struct optimized_kprobe *op, int dirty)
 339 {
 340         if (op->optinsn.insn) {
 341                 free_optinsn_slot(op->optinsn.insn, dirty);
 342                 op->optinsn.insn = NULL;
 343                 op->optinsn.size = 0;
 344         }
 345 }
 346
 347 void arch_remove_optimized_kprobe(struct optimized_kprobe *op)
 348 {
 349         __arch_remove_optimized_kprobe(op, 1);
 350 }
 351
 352 /*
 353  * Copy replacing target instructions
 354  * Target instructions MUST be relocatable (checked inside)
 355  * This is called when new aggr(opt)probe is allocated or reused.
 356  */
 357 int arch_prepare_optimized_kprobe(struct optimized_kprobe *op,
 358                                   struct kprobe *__unused)
 359 {
 360         u8 *buf;
 361         int ret;
 362         long rel;
 363
 364         if (!can_optimize((unsigned long)op->kp.addr))
 365                 return -EILSEQ;
 366
 367         op->optinsn.insn = get_optinsn_slot();
 368         if (!op->optinsn.insn)
 369                 return -ENOMEM;
 370
 371         /*
 372          * Verify if the address gap is in 2GB range, because this uses
 373          * a relative jump.
 374          */
 375         rel = (long)op->optinsn.insn - (long)op->kp.addr + RELATIVEJUMP_SIZE;
 376         if (abs(rel) > 0x7fffffff) {
 377                 __arch_remove_optimized_kprobe(op, 0);
 378                 return -ERANGE;
 379         }
 380
 381         buf = (u8 *)op->optinsn.insn;
 382         set_memory_rw((unsigned long)buf & PAGE_MASK, 1);
 383
 384         /* Copy instructions into the out-of-line buffer */
 385         ret = copy_optimized_instructions(buf + TMPL_END_IDX, op->kp.addr);
 386         if (ret < 0) {
 387                 __arch_remove_optimized_kprobe(op, 0);
 388                 return ret;
 389         }
 390         op->optinsn.size = ret;
 391
 392         /* Copy arch-dep-instance from template */
 393         memcpy(buf, &optprobe_template_entry, TMPL_END_IDX);
 394
 395         /* Set probe information */
 396         synthesize_set_arg1(buf + TMPL_MOVE_IDX, (unsigned long)op);
 397
 398         /* Set probe function call */
 399         synthesize_relcall(buf + TMPL_CALL_IDX, optimized_callback);
 400
 401         /* Set returning jmp instruction at the tail of out-of-line buffer */
 402         synthesize_reljump(buf + TMPL_END_IDX + op->optinsn.size,
 403                            (u8 *)op->kp.addr + op->optinsn.size);
 404
 405         set_memory_ro((unsigned long)buf & PAGE_MASK, 1);
 406
 407         flush_icache_range((unsigned long) buf,
 408                            (unsigned long) buf + TMPL_END_IDX +
 409                            op->optinsn.size + RELATIVEJUMP_SIZE);
 410         return 0;
 411 }
 412
 413 /*
 414  * Replace breakpoints (int3) with relative jumps.
 415  * Caller must call with locking kprobe_mutex and text_mutex.
 416  */
 417 void arch_optimize_kprobes(struct list_head *oplist)
 418 {
 419         struct optimized_kprobe *op, *tmp;
 420         u8 insn_buf[RELATIVEJUMP_SIZE];
 421
 422         list_for_each_entry_safe(op, tmp, oplist, list) {
 423                 s32 rel = (s32)((long)op->optinsn.insn -
 424                         ((long)op->kp.addr + RELATIVEJUMP_SIZE));
 425
 426                 WARN_ON(kprobe_disabled(&op->kp));
 427
 428                 /* Backup instructions which will be replaced by jump address */
 429                 memcpy(op->optinsn.copied_insn, op->kp.addr + INT3_SIZE,
 430                        RELATIVE_ADDR_SIZE);
 431
 432                 insn_buf[0] = RELATIVEJUMP_OPCODE;
 433                 *(s32 *)(&insn_buf[1]) = rel;
 434
 435                 text_poke_bp(op->kp.addr, insn_buf, RELATIVEJUMP_SIZE,
 436                              op->optinsn.insn);
 437
 438                 list_del_init(&op->list);
 439         }
 440 }
 441
 442 /* Replace a relative jump with a breakpoint (int3).  */
 443 void arch_unoptimize_kprobe(struct optimized_kprobe *op)
 444 {
 445         u8 insn_buf[RELATIVEJUMP_SIZE];
 446
 447         /* Set int3 to first byte for kprobes */
 448         insn_buf[0] = BREAKPOINT_INSTRUCTION;
 449         memcpy(insn_buf + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE);
 450         text_poke_bp(op->kp.addr, insn_buf, RELATIVEJUMP_SIZE,
 451                      op->optinsn.insn);
 452 }
 453
 454 /*
 455  * Recover original instructions and breakpoints from relative jumps.
 456  * Caller must call with locking kprobe_mutex.
 457  */
 458 extern void arch_unoptimize_kprobes(struct list_head *oplist,
 459                                     struct list_head *done_list)
 460 {
 461         struct optimized_kprobe *op, *tmp;
 462
 463         list_for_each_entry_safe(op, tmp, oplist, list) {
 464                 arch_unoptimize_kprobe(op);
 465                 list_move(&op->list, done_list);
 466         }
 467 }
 468
 469 int setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter)
 470 {
 471         struct optimized_kprobe *op;
 472
 473         if (p->flags & KPROBE_FLAG_OPTIMIZED) {
 474                 /* This kprobe is really able to run optimized path. */
 475                 op = container_of(p, struct optimized_kprobe, kp);
 476                 /* Detour through copied instructions */
 477                 regs->ip = (unsigned long)op->optinsn.insn + TMPL_END_IDX;
 478                 if (!reenter)
 479                         reset_current_kprobe();
 480                 preempt_enable_no_resched();
 481                 return 1;
 482         }
 483         return 0;
 484 }
 485 NOKPROBE_SYMBOL(setup_detour_execution);