1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Kernel Probes (KProbes)
5 * Copyright (C) IBM Corporation, 2002, 2004
7 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
8 * Probes initial implementation ( includes contributions from
10 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
11 * interface to access function arguments.
12 * 2004-Nov Ananth N Mavinakayanahalli <ananth@in.ibm.com> kprobes port
16 #include <linux/kprobes.h>
17 #include <linux/ptrace.h>
18 #include <linux/preempt.h>
19 #include <linux/extable.h>
20 #include <linux/kdebug.h>
21 #include <linux/slab.h>
22 #include <asm/code-patching.h>
23 #include <asm/cacheflush.h>
24 #include <asm/sstep.h>
25 #include <asm/sections.h>
27 #include <linux/uaccess.h>
29 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
30 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
32 struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}};
34 bool arch_within_kprobe_blacklist(unsigned long addr)
36 return (addr >= (unsigned long)__kprobes_text_start &&
37 addr < (unsigned long)__kprobes_text_end) ||
38 (addr >= (unsigned long)_stext &&
39 addr < (unsigned long)__head_end);
42 kprobe_opcode_t *kprobe_lookup_name(const char *name, unsigned int offset)
44 kprobe_opcode_t *addr = NULL;
46 #ifdef PPC64_ELF_ABI_v2
47 /* PPC64 ABIv2 needs local entry point */
48 addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
49 if (addr && !offset) {
50 #ifdef CONFIG_KPROBES_ON_FTRACE
53 * Per livepatch.h, ftrace location is always within the first
54 * 16 bytes of a function on powerpc with -mprofile-kernel.
56 faddr = ftrace_location_range((unsigned long)addr,
57 (unsigned long)addr + 16);
59 addr = (kprobe_opcode_t *)faddr;
62 addr = (kprobe_opcode_t *)ppc_function_entry(addr);
64 #elif defined(PPC64_ELF_ABI_v1)
66 * 64bit powerpc ABIv1 uses function descriptors:
67 * - Check for the dot variant of the symbol first.
68 * - If that fails, try looking up the symbol provided.
70 * This ensures we always get to the actual symbol and not
73 * Also handle <module:symbol> format.
75 char dot_name[MODULE_NAME_LEN + 1 + KSYM_NAME_LEN];
76 bool dot_appended = false;
81 if ((c = strnchr(name, MODULE_NAME_LEN, ':')) != NULL) {
84 memcpy(dot_name, name, len);
88 if (*c != '\0' && *c != '.') {
89 dot_name[len++] = '.';
92 ret = strscpy(dot_name + len, c, KSYM_NAME_LEN);
94 addr = (kprobe_opcode_t *)kallsyms_lookup_name(dot_name);
96 /* Fallback to the original non-dot symbol lookup */
97 if (!addr && dot_appended)
98 addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
100 addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
106 int arch_prepare_kprobe(struct kprobe *p)
110 struct ppc_inst insn = ppc_inst_read((struct ppc_inst *)p->addr);
112 if ((unsigned long)p->addr & 0x03) {
113 printk("Attempt to register kprobe at an unaligned address\n");
115 } else if (IS_MTMSRD(insn) || IS_RFID(insn) || IS_RFI(insn)) {
116 printk("Cannot register a kprobe on rfi/rfid or mtmsr[d]\n");
118 } else if ((unsigned long)p->addr & ~PAGE_MASK &&
119 ppc_inst_prefixed(ppc_inst_read((struct ppc_inst *)(p->addr - 1)))) {
120 printk("Cannot register a kprobe on the second word of prefixed instruction\n");
124 prev = get_kprobe(p->addr - 1);
125 preempt_enable_no_resched();
127 ppc_inst_prefixed(ppc_inst_read((struct ppc_inst *)prev->ainsn.insn))) {
128 printk("Cannot register a kprobe on the second word of prefixed instruction\n");
132 /* insn must be on a special executable page on ppc64. This is
133 * not explicitly required on ppc32 (right now), but it doesn't hurt */
135 p->ainsn.insn = get_insn_slot();
141 patch_instruction((struct ppc_inst *)p->ainsn.insn, insn);
142 p->opcode = ppc_inst_val(insn);
145 p->ainsn.boostable = 0;
148 NOKPROBE_SYMBOL(arch_prepare_kprobe);
150 void arch_arm_kprobe(struct kprobe *p)
152 patch_instruction((struct ppc_inst *)p->addr, ppc_inst(BREAKPOINT_INSTRUCTION));
154 NOKPROBE_SYMBOL(arch_arm_kprobe);
156 void arch_disarm_kprobe(struct kprobe *p)
158 patch_instruction((struct ppc_inst *)p->addr, ppc_inst(p->opcode));
160 NOKPROBE_SYMBOL(arch_disarm_kprobe);
162 void arch_remove_kprobe(struct kprobe *p)
165 free_insn_slot(p->ainsn.insn, 0);
166 p->ainsn.insn = NULL;
169 NOKPROBE_SYMBOL(arch_remove_kprobe);
171 static nokprobe_inline void prepare_singlestep(struct kprobe *p, struct pt_regs *regs)
173 enable_single_step(regs);
176 * On powerpc we should single step on the original
177 * instruction even if the probed insn is a trap
178 * variant as values in regs could play a part in
179 * if the trap is taken or not
181 regs->nip = (unsigned long)p->ainsn.insn;
184 static nokprobe_inline void save_previous_kprobe(struct kprobe_ctlblk *kcb)
186 kcb->prev_kprobe.kp = kprobe_running();
187 kcb->prev_kprobe.status = kcb->kprobe_status;
188 kcb->prev_kprobe.saved_msr = kcb->kprobe_saved_msr;
191 static nokprobe_inline void restore_previous_kprobe(struct kprobe_ctlblk *kcb)
193 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
194 kcb->kprobe_status = kcb->prev_kprobe.status;
195 kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr;
198 static nokprobe_inline void set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
199 struct kprobe_ctlblk *kcb)
201 __this_cpu_write(current_kprobe, p);
202 kcb->kprobe_saved_msr = regs->msr;
205 bool arch_kprobe_on_func_entry(unsigned long offset)
207 #ifdef PPC64_ELF_ABI_v2
208 #ifdef CONFIG_KPROBES_ON_FTRACE
218 void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
220 ri->ret_addr = (kprobe_opcode_t *)regs->link;
223 /* Replace the return addr with trampoline addr */
224 regs->link = (unsigned long)kretprobe_trampoline;
226 NOKPROBE_SYMBOL(arch_prepare_kretprobe);
228 static int try_to_emulate(struct kprobe *p, struct pt_regs *regs)
231 struct ppc_inst insn = ppc_inst_read((struct ppc_inst *)p->ainsn.insn);
233 /* regs->nip is also adjusted if emulate_step returns 1 */
234 ret = emulate_step(regs, insn);
237 * Once this instruction has been boosted
238 * successfully, set the boostable flag
240 if (unlikely(p->ainsn.boostable == 0))
241 p->ainsn.boostable = 1;
242 } else if (ret < 0) {
244 * We don't allow kprobes on mtmsr(d)/rfi(d), etc.
245 * So, we should never get here... but, its still
246 * good to catch them, just in case...
248 printk("Can't step on instruction %s\n", ppc_inst_as_str(insn));
252 * If we haven't previously emulated this instruction, then it
253 * can't be boosted. Note it down so we don't try to do so again.
255 * If, however, we had emulated this instruction in the past,
256 * then this is just an error with the current run (for
257 * instance, exceptions due to a load/store). We return 0 so
258 * that this is now single-stepped, but continue to try
259 * emulating it in subsequent probe hits.
261 if (unlikely(p->ainsn.boostable != 1))
262 p->ainsn.boostable = -1;
267 NOKPROBE_SYMBOL(try_to_emulate);
269 int kprobe_handler(struct pt_regs *regs)
273 unsigned int *addr = (unsigned int *)regs->nip;
274 struct kprobe_ctlblk *kcb;
279 if (!IS_ENABLED(CONFIG_BOOKE) &&
280 (!(regs->msr & MSR_IR) || !(regs->msr & MSR_DR)))
284 * We don't want to be preempted for the entire
285 * duration of kprobe processing
288 kcb = get_kprobe_ctlblk();
290 p = get_kprobe(addr);
294 if (get_kernel_nofault(instr, addr))
297 if (instr != BREAKPOINT_INSTRUCTION) {
299 * PowerPC has multiple variants of the "trap"
300 * instruction. If the current instruction is a
301 * trap variant, it could belong to someone else
306 * The breakpoint instruction was removed right
307 * after we hit it. Another cpu has removed
308 * either a probepoint or a debugger breakpoint
309 * at this address. In either case, no further
310 * handling of this interrupt is appropriate.
314 /* Not one of ours: let kernel handle it */
318 /* Check we're not actually recursing */
319 if (kprobe_running()) {
320 kprobe_opcode_t insn = *p->ainsn.insn;
321 if (kcb->kprobe_status == KPROBE_HIT_SS && is_trap(insn)) {
322 /* Turn off 'trace' bits */
323 regs->msr &= ~MSR_SINGLESTEP;
324 regs->msr |= kcb->kprobe_saved_msr;
329 * We have reentered the kprobe_handler(), since another probe
330 * was hit while within the handler. We here save the original
331 * kprobes variables and just single step on the instruction of
332 * the new probe without calling any user handlers.
334 save_previous_kprobe(kcb);
335 set_current_kprobe(p, regs, kcb);
336 kprobes_inc_nmissed_count(p);
337 kcb->kprobe_status = KPROBE_REENTER;
338 if (p->ainsn.boostable >= 0) {
339 ret = try_to_emulate(p, regs);
342 restore_previous_kprobe(kcb);
343 preempt_enable_no_resched();
347 prepare_singlestep(p, regs);
351 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
352 set_current_kprobe(p, regs, kcb);
353 if (p->pre_handler && p->pre_handler(p, regs)) {
354 /* handler changed execution path, so skip ss setup */
355 reset_current_kprobe();
356 preempt_enable_no_resched();
360 if (p->ainsn.boostable >= 0) {
361 ret = try_to_emulate(p, regs);
365 p->post_handler(p, regs, 0);
367 kcb->kprobe_status = KPROBE_HIT_SSDONE;
368 reset_current_kprobe();
369 preempt_enable_no_resched();
373 prepare_singlestep(p, regs);
374 kcb->kprobe_status = KPROBE_HIT_SS;
378 preempt_enable_no_resched();
381 NOKPROBE_SYMBOL(kprobe_handler);
384 * Function return probe trampoline:
385 * - init_kprobes() establishes a probepoint here
386 * - When the probed function returns, this probe
387 * causes the handlers to fire
389 asm(".global kretprobe_trampoline\n"
390 ".type kretprobe_trampoline, @function\n"
391 "kretprobe_trampoline:\n"
394 ".size kretprobe_trampoline, .-kretprobe_trampoline\n");
397 * Called when the probe at kretprobe trampoline is hit
399 static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
401 unsigned long orig_ret_address;
403 orig_ret_address = __kretprobe_trampoline_handler(regs, &kretprobe_trampoline, NULL);
405 * We get here through one of two paths:
406 * 1. by taking a trap -> kprobe_handler() -> here
407 * 2. by optprobe branch -> optimized_callback() -> opt_pre_handler() -> here
409 * When going back through (1), we need regs->nip to be setup properly
410 * as it is used to determine the return address from the trap.
411 * For (2), since nip is not honoured with optprobes, we instead setup
412 * the link register properly so that the subsequent 'blr' in
413 * kretprobe_trampoline jumps back to the right instruction.
415 * For nip, we should set the address to the previous instruction since
416 * we end up emulating it in kprobe_handler(), which increments the nip
419 regs->nip = orig_ret_address - 4;
420 regs->link = orig_ret_address;
424 NOKPROBE_SYMBOL(trampoline_probe_handler);
427 * Called after single-stepping. p->addr is the address of the
428 * instruction whose first byte has been replaced by the "breakpoint"
429 * instruction. To avoid the SMP problems that can occur when we
430 * temporarily put back the original opcode to single-step, we
431 * single-stepped a copy of the instruction. The address of this
432 * copy is p->ainsn.insn.
434 int kprobe_post_handler(struct pt_regs *regs)
437 struct kprobe *cur = kprobe_running();
438 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
440 if (!cur || user_mode(regs))
443 len = ppc_inst_len(ppc_inst_read((struct ppc_inst *)cur->ainsn.insn));
444 /* make sure we got here for instruction we have a kprobe on */
445 if (((unsigned long)cur->ainsn.insn + len) != regs->nip)
448 if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
449 kcb->kprobe_status = KPROBE_HIT_SSDONE;
450 cur->post_handler(cur, regs, 0);
453 /* Adjust nip to after the single-stepped instruction */
454 regs->nip = (unsigned long)cur->addr + len;
455 regs->msr |= kcb->kprobe_saved_msr;
457 /*Restore back the original saved kprobes variables and continue. */
458 if (kcb->kprobe_status == KPROBE_REENTER) {
459 restore_previous_kprobe(kcb);
462 reset_current_kprobe();
464 preempt_enable_no_resched();
467 * if somebody else is singlestepping across a probe point, msr
468 * will have DE/SE set, in which case, continue the remaining processing
469 * of do_debug, as if this is not a probe hit.
471 if (regs->msr & MSR_SINGLESTEP)
476 NOKPROBE_SYMBOL(kprobe_post_handler);
478 int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
480 struct kprobe *cur = kprobe_running();
481 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
482 const struct exception_table_entry *entry;
484 switch(kcb->kprobe_status) {
488 * We are here because the instruction being single
489 * stepped caused a page fault. We reset the current
490 * kprobe and the nip points back to the probe address
491 * and allow the page fault handler to continue as a
494 regs->nip = (unsigned long)cur->addr;
495 regs->msr &= ~MSR_SINGLESTEP; /* Turn off 'trace' bits */
496 regs->msr |= kcb->kprobe_saved_msr;
497 if (kcb->kprobe_status == KPROBE_REENTER)
498 restore_previous_kprobe(kcb);
500 reset_current_kprobe();
501 preempt_enable_no_resched();
503 case KPROBE_HIT_ACTIVE:
504 case KPROBE_HIT_SSDONE:
506 * We increment the nmissed count for accounting,
507 * we can also use npre/npostfault count for accounting
508 * these specific fault cases.
510 kprobes_inc_nmissed_count(cur);
513 * We come here because instructions in the pre/post
514 * handler caused the page_fault, this could happen
515 * if handler tries to access user space by
516 * copy_from_user(), get_user() etc. Let the
517 * user-specified handler try to fix it first.
519 if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))
523 * In case the user-specified fault handler returned
524 * zero, try to fix up.
526 if ((entry = search_exception_tables(regs->nip)) != NULL) {
527 regs->nip = extable_fixup(entry);
532 * fixup_exception() could not handle it,
533 * Let do_page_fault() fix it.
541 NOKPROBE_SYMBOL(kprobe_fault_handler);
543 unsigned long arch_deref_entry_point(void *entry)
545 #ifdef PPC64_ELF_ABI_v1
546 if (!kernel_text_address((unsigned long)entry))
547 return ppc_global_function_entry(entry);
550 return (unsigned long)entry;
552 NOKPROBE_SYMBOL(arch_deref_entry_point);
554 static struct kprobe trampoline_p = {
555 .addr = (kprobe_opcode_t *) &kretprobe_trampoline,
556 .pre_handler = trampoline_probe_handler
559 int __init arch_init_kprobes(void)
561 return register_kprobe(&trampoline_p);
564 int arch_trampoline_kprobe(struct kprobe *p)
566 if (p->addr == (kprobe_opcode_t *)&kretprobe_trampoline)
571 NOKPROBE_SYMBOL(arch_trampoline_kprobe);