1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Kernel Probes (KProbes)
5 * Copyright (C) IBM Corporation, 2002, 2004
7 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
8 * Probes initial implementation ( includes contributions from
10 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
11 * interface to access function arguments.
12 * 2004-Nov Ananth N Mavinakayanahalli <ananth@in.ibm.com> kprobes port
16 #include <linux/kprobes.h>
17 #include <linux/ptrace.h>
18 #include <linux/preempt.h>
19 #include <linux/extable.h>
20 #include <linux/kdebug.h>
21 #include <linux/slab.h>
22 #include <linux/moduleloader.h>
23 #include <asm/code-patching.h>
24 #include <asm/cacheflush.h>
25 #include <asm/sstep.h>
26 #include <asm/sections.h>
28 #include <asm/set_memory.h>
29 #include <linux/uaccess.h>
31 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
32 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
34 struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}};
36 bool arch_within_kprobe_blacklist(unsigned long addr)
38 return (addr >= (unsigned long)__kprobes_text_start &&
39 addr < (unsigned long)__kprobes_text_end) ||
40 (addr >= (unsigned long)_stext &&
41 addr < (unsigned long)__head_end);
44 kprobe_opcode_t *kprobe_lookup_name(const char *name, unsigned int offset)
46 kprobe_opcode_t *addr = NULL;
48 #ifdef CONFIG_PPC64_ELF_ABI_V2
49 /* PPC64 ABIv2 needs local entry point */
50 addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
51 if (addr && !offset) {
52 #ifdef CONFIG_KPROBES_ON_FTRACE
55 * Per livepatch.h, ftrace location is always within the first
56 * 16 bytes of a function on powerpc with -mprofile-kernel.
58 faddr = ftrace_location_range((unsigned long)addr,
59 (unsigned long)addr + 16);
61 addr = (kprobe_opcode_t *)faddr;
64 addr = (kprobe_opcode_t *)ppc_function_entry(addr);
66 #elif defined(CONFIG_PPC64_ELF_ABI_V1)
68 * 64bit powerpc ABIv1 uses function descriptors:
69 * - Check for the dot variant of the symbol first.
70 * - If that fails, try looking up the symbol provided.
72 * This ensures we always get to the actual symbol and not
75 * Also handle <module:symbol> format.
77 char dot_name[MODULE_NAME_LEN + 1 + KSYM_NAME_LEN];
78 bool dot_appended = false;
83 if ((c = strnchr(name, MODULE_NAME_LEN, ':')) != NULL) {
86 memcpy(dot_name, name, len);
90 if (*c != '\0' && *c != '.') {
91 dot_name[len++] = '.';
94 ret = strscpy(dot_name + len, c, KSYM_NAME_LEN);
96 addr = (kprobe_opcode_t *)kallsyms_lookup_name(dot_name);
98 /* Fallback to the original non-dot symbol lookup */
99 if (!addr && dot_appended)
100 addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
102 addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
108 static bool arch_kprobe_on_func_entry(unsigned long offset)
110 #ifdef CONFIG_PPC64_ELF_ABI_V2
111 #ifdef CONFIG_KPROBES_ON_FTRACE
121 /* XXX try and fold the magic of kprobe_lookup_name() in this */
122 kprobe_opcode_t *arch_adjust_kprobe_addr(unsigned long addr, unsigned long offset,
125 *on_func_entry = arch_kprobe_on_func_entry(offset);
126 return (kprobe_opcode_t *)(addr + offset);
129 void *alloc_insn_page(void)
133 page = module_alloc(PAGE_SIZE);
137 if (strict_module_rwx_enabled()) {
138 set_memory_ro((unsigned long)page, 1);
139 set_memory_x((unsigned long)page, 1);
144 int arch_prepare_kprobe(struct kprobe *p)
148 ppc_inst_t insn = ppc_inst_read(p->addr);
150 if ((unsigned long)p->addr & 0x03) {
151 printk("Attempt to register kprobe at an unaligned address\n");
153 } else if (!can_single_step(ppc_inst_val(insn))) {
154 printk("Cannot register a kprobe on instructions that can't be single stepped\n");
156 } else if ((unsigned long)p->addr & ~PAGE_MASK &&
157 ppc_inst_prefixed(ppc_inst_read(p->addr - 1))) {
158 printk("Cannot register a kprobe on the second word of prefixed instruction\n");
162 prev = get_kprobe(p->addr - 1);
163 preempt_enable_no_resched();
166 * When prev is a ftrace-based kprobe, we don't have an insn, and it
167 * doesn't probe for prefixed instruction.
169 if (prev && !kprobe_ftrace(prev) &&
170 ppc_inst_prefixed(ppc_inst_read(prev->ainsn.insn))) {
171 printk("Cannot register a kprobe on the second word of prefixed instruction\n");
175 /* insn must be on a special executable page on ppc64. This is
176 * not explicitly required on ppc32 (right now), but it doesn't hurt */
178 p->ainsn.insn = get_insn_slot();
184 patch_instruction(p->ainsn.insn, insn);
185 p->opcode = ppc_inst_val(insn);
188 p->ainsn.boostable = 0;
191 NOKPROBE_SYMBOL(arch_prepare_kprobe);
193 void arch_arm_kprobe(struct kprobe *p)
195 WARN_ON_ONCE(patch_instruction(p->addr, ppc_inst(BREAKPOINT_INSTRUCTION)));
197 NOKPROBE_SYMBOL(arch_arm_kprobe);
199 void arch_disarm_kprobe(struct kprobe *p)
201 WARN_ON_ONCE(patch_instruction(p->addr, ppc_inst(p->opcode)));
203 NOKPROBE_SYMBOL(arch_disarm_kprobe);
205 void arch_remove_kprobe(struct kprobe *p)
208 free_insn_slot(p->ainsn.insn, 0);
209 p->ainsn.insn = NULL;
212 NOKPROBE_SYMBOL(arch_remove_kprobe);
214 static nokprobe_inline void prepare_singlestep(struct kprobe *p, struct pt_regs *regs)
216 enable_single_step(regs);
219 * On powerpc we should single step on the original
220 * instruction even if the probed insn is a trap
221 * variant as values in regs could play a part in
222 * if the trap is taken or not
224 regs_set_return_ip(regs, (unsigned long)p->ainsn.insn);
227 static nokprobe_inline void save_previous_kprobe(struct kprobe_ctlblk *kcb)
229 kcb->prev_kprobe.kp = kprobe_running();
230 kcb->prev_kprobe.status = kcb->kprobe_status;
231 kcb->prev_kprobe.saved_msr = kcb->kprobe_saved_msr;
234 static nokprobe_inline void restore_previous_kprobe(struct kprobe_ctlblk *kcb)
236 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
237 kcb->kprobe_status = kcb->prev_kprobe.status;
238 kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr;
241 static nokprobe_inline void set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
242 struct kprobe_ctlblk *kcb)
244 __this_cpu_write(current_kprobe, p);
245 kcb->kprobe_saved_msr = regs->msr;
248 void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
250 ri->ret_addr = (kprobe_opcode_t *)regs->link;
253 /* Replace the return addr with trampoline addr */
254 regs->link = (unsigned long)__kretprobe_trampoline;
256 NOKPROBE_SYMBOL(arch_prepare_kretprobe);
258 static int try_to_emulate(struct kprobe *p, struct pt_regs *regs)
261 ppc_inst_t insn = ppc_inst_read(p->ainsn.insn);
263 /* regs->nip is also adjusted if emulate_step returns 1 */
264 ret = emulate_step(regs, insn);
267 * Once this instruction has been boosted
268 * successfully, set the boostable flag
270 if (unlikely(p->ainsn.boostable == 0))
271 p->ainsn.boostable = 1;
272 } else if (ret < 0) {
274 * We don't allow kprobes on mtmsr(d)/rfi(d), etc.
275 * So, we should never get here... but, its still
276 * good to catch them, just in case...
278 printk("Can't step on instruction %08lx\n", ppc_inst_as_ulong(insn));
282 * If we haven't previously emulated this instruction, then it
283 * can't be boosted. Note it down so we don't try to do so again.
285 * If, however, we had emulated this instruction in the past,
286 * then this is just an error with the current run (for
287 * instance, exceptions due to a load/store). We return 0 so
288 * that this is now single-stepped, but continue to try
289 * emulating it in subsequent probe hits.
291 if (unlikely(p->ainsn.boostable != 1))
292 p->ainsn.boostable = -1;
297 NOKPROBE_SYMBOL(try_to_emulate);
299 int kprobe_handler(struct pt_regs *regs)
303 unsigned int *addr = (unsigned int *)regs->nip;
304 struct kprobe_ctlblk *kcb;
309 if (!IS_ENABLED(CONFIG_BOOKE) &&
310 (!(regs->msr & MSR_IR) || !(regs->msr & MSR_DR)))
314 * We don't want to be preempted for the entire
315 * duration of kprobe processing
318 kcb = get_kprobe_ctlblk();
320 p = get_kprobe(addr);
324 if (get_kernel_nofault(instr, addr))
327 if (instr != BREAKPOINT_INSTRUCTION) {
329 * PowerPC has multiple variants of the "trap"
330 * instruction. If the current instruction is a
331 * trap variant, it could belong to someone else
336 * The breakpoint instruction was removed right
337 * after we hit it. Another cpu has removed
338 * either a probepoint or a debugger breakpoint
339 * at this address. In either case, no further
340 * handling of this interrupt is appropriate.
344 /* Not one of ours: let kernel handle it */
348 /* Check we're not actually recursing */
349 if (kprobe_running()) {
350 kprobe_opcode_t insn = *p->ainsn.insn;
351 if (kcb->kprobe_status == KPROBE_HIT_SS && is_trap(insn)) {
352 /* Turn off 'trace' bits */
353 regs_set_return_msr(regs,
354 (regs->msr & ~MSR_SINGLESTEP) |
355 kcb->kprobe_saved_msr);
360 * We have reentered the kprobe_handler(), since another probe
361 * was hit while within the handler. We here save the original
362 * kprobes variables and just single step on the instruction of
363 * the new probe without calling any user handlers.
365 save_previous_kprobe(kcb);
366 set_current_kprobe(p, regs, kcb);
367 kprobes_inc_nmissed_count(p);
368 kcb->kprobe_status = KPROBE_REENTER;
369 if (p->ainsn.boostable >= 0) {
370 ret = try_to_emulate(p, regs);
373 restore_previous_kprobe(kcb);
374 preempt_enable_no_resched();
378 prepare_singlestep(p, regs);
382 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
383 set_current_kprobe(p, regs, kcb);
384 if (p->pre_handler && p->pre_handler(p, regs)) {
385 /* handler changed execution path, so skip ss setup */
386 reset_current_kprobe();
387 preempt_enable_no_resched();
391 if (p->ainsn.boostable >= 0) {
392 ret = try_to_emulate(p, regs);
396 p->post_handler(p, regs, 0);
398 kcb->kprobe_status = KPROBE_HIT_SSDONE;
399 reset_current_kprobe();
400 preempt_enable_no_resched();
404 prepare_singlestep(p, regs);
405 kcb->kprobe_status = KPROBE_HIT_SS;
409 preempt_enable_no_resched();
412 NOKPROBE_SYMBOL(kprobe_handler);
415 * Function return probe trampoline:
416 * - init_kprobes() establishes a probepoint here
417 * - When the probed function returns, this probe
418 * causes the handlers to fire
420 asm(".global __kretprobe_trampoline\n"
421 ".type __kretprobe_trampoline, @function\n"
422 "__kretprobe_trampoline:\n"
425 ".size __kretprobe_trampoline, .-__kretprobe_trampoline\n");
428 * Called when the probe at kretprobe trampoline is hit
430 static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
432 unsigned long orig_ret_address;
434 orig_ret_address = __kretprobe_trampoline_handler(regs, NULL);
436 * We get here through one of two paths:
437 * 1. by taking a trap -> kprobe_handler() -> here
438 * 2. by optprobe branch -> optimized_callback() -> opt_pre_handler() -> here
440 * When going back through (1), we need regs->nip to be setup properly
441 * as it is used to determine the return address from the trap.
442 * For (2), since nip is not honoured with optprobes, we instead setup
443 * the link register properly so that the subsequent 'blr' in
444 * __kretprobe_trampoline jumps back to the right instruction.
446 * For nip, we should set the address to the previous instruction since
447 * we end up emulating it in kprobe_handler(), which increments the nip
450 regs_set_return_ip(regs, orig_ret_address - 4);
451 regs->link = orig_ret_address;
455 NOKPROBE_SYMBOL(trampoline_probe_handler);
458 * Called after single-stepping. p->addr is the address of the
459 * instruction whose first byte has been replaced by the "breakpoint"
460 * instruction. To avoid the SMP problems that can occur when we
461 * temporarily put back the original opcode to single-step, we
462 * single-stepped a copy of the instruction. The address of this
463 * copy is p->ainsn.insn.
465 int kprobe_post_handler(struct pt_regs *regs)
468 struct kprobe *cur = kprobe_running();
469 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
471 if (!cur || user_mode(regs))
474 len = ppc_inst_len(ppc_inst_read(cur->ainsn.insn));
475 /* make sure we got here for instruction we have a kprobe on */
476 if (((unsigned long)cur->ainsn.insn + len) != regs->nip)
479 if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
480 kcb->kprobe_status = KPROBE_HIT_SSDONE;
481 cur->post_handler(cur, regs, 0);
484 /* Adjust nip to after the single-stepped instruction */
485 regs_set_return_ip(regs, (unsigned long)cur->addr + len);
486 regs_set_return_msr(regs, regs->msr | kcb->kprobe_saved_msr);
488 /*Restore back the original saved kprobes variables and continue. */
489 if (kcb->kprobe_status == KPROBE_REENTER) {
490 restore_previous_kprobe(kcb);
493 reset_current_kprobe();
495 preempt_enable_no_resched();
498 * if somebody else is singlestepping across a probe point, msr
499 * will have DE/SE set, in which case, continue the remaining processing
500 * of do_debug, as if this is not a probe hit.
502 if (regs->msr & MSR_SINGLESTEP)
507 NOKPROBE_SYMBOL(kprobe_post_handler);
509 int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
511 struct kprobe *cur = kprobe_running();
512 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
513 const struct exception_table_entry *entry;
515 switch(kcb->kprobe_status) {
519 * We are here because the instruction being single
520 * stepped caused a page fault. We reset the current
521 * kprobe and the nip points back to the probe address
522 * and allow the page fault handler to continue as a
525 regs_set_return_ip(regs, (unsigned long)cur->addr);
526 /* Turn off 'trace' bits */
527 regs_set_return_msr(regs,
528 (regs->msr & ~MSR_SINGLESTEP) |
529 kcb->kprobe_saved_msr);
530 if (kcb->kprobe_status == KPROBE_REENTER)
531 restore_previous_kprobe(kcb);
533 reset_current_kprobe();
534 preempt_enable_no_resched();
536 case KPROBE_HIT_ACTIVE:
537 case KPROBE_HIT_SSDONE:
539 * In case the user-specified fault handler returned
540 * zero, try to fix up.
542 if ((entry = search_exception_tables(regs->nip)) != NULL) {
543 regs_set_return_ip(regs, extable_fixup(entry));
548 * fixup_exception() could not handle it,
549 * Let do_page_fault() fix it.
557 NOKPROBE_SYMBOL(kprobe_fault_handler);
559 static struct kprobe trampoline_p = {
560 .addr = (kprobe_opcode_t *) &__kretprobe_trampoline,
561 .pre_handler = trampoline_probe_handler
564 int __init arch_init_kprobes(void)
566 return register_kprobe(&trampoline_p);
569 int arch_trampoline_kprobe(struct kprobe *p)
571 if (p->addr == (kprobe_opcode_t *)&__kretprobe_trampoline)
576 NOKPROBE_SYMBOL(arch_trampoline_kprobe);