2 * Kernel Probes (KProbes)
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 * Copyright IBM Corp. 2002, 2006
20 * s390 port, used ppc64 as template. Mike Grundy <grundym@us.ibm.com>
23 #include <linux/kprobes.h>
24 #include <linux/ptrace.h>
25 #include <linux/preempt.h>
26 #include <linux/stop_machine.h>
27 #include <linux/kdebug.h>
28 #include <linux/uaccess.h>
29 #include <linux/extable.h>
30 #include <linux/module.h>
31 #include <linux/slab.h>
32 #include <linux/hardirq.h>
33 #include <linux/ftrace.h>
34 #include <asm/set_memory.h>
35 #include <asm/sections.h>
36 #include <linux/uaccess.h>
39 DEFINE_PER_CPU(struct kprobe *, current_kprobe);
40 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
42 struct kretprobe_blackpoint kretprobe_blacklist[] = { };
44 DEFINE_INSN_CACHE_OPS(dmainsn);
46 static void *alloc_dmainsn_page(void)
50 page = (void *) __get_free_page(GFP_KERNEL | GFP_DMA);
52 set_memory_x((unsigned long) page, 1);
56 static void free_dmainsn_page(void *page)
58 set_memory_nx((unsigned long) page, 1);
59 free_page((unsigned long)page);
62 struct kprobe_insn_cache kprobe_dmainsn_slots = {
63 .mutex = __MUTEX_INITIALIZER(kprobe_dmainsn_slots.mutex),
64 .alloc = alloc_dmainsn_page,
65 .free = free_dmainsn_page,
66 .pages = LIST_HEAD_INIT(kprobe_dmainsn_slots.pages),
67 .insn_size = MAX_INSN_SIZE,
70 static void copy_instruction(struct kprobe *p)
72 unsigned long ip = (unsigned long) p->addr;
76 if (ftrace_location(ip) == ip) {
78 * If kprobes patches the instruction that is morphed by
79 * ftrace make sure that kprobes always sees the branch
80 * "jg .+24" that skips the mcount block or the "brcl 0,0"
81 * in case of hotpatch.
83 ftrace_generate_nop_insn((struct ftrace_insn *)p->ainsn.insn);
84 p->ainsn.is_ftrace_insn = 1;
86 memcpy(p->ainsn.insn, p->addr, insn_length(*p->addr >> 8));
87 p->opcode = p->ainsn.insn[0];
88 if (!probe_is_insn_relative_long(p->ainsn.insn))
91 * For pc-relative instructions in RIL-b or RIL-c format patch the
92 * RI2 displacement field. We have already made sure that the insn
93 * slot for the patched instruction is within the same 2GB area
94 * as the original instruction (either kernel image or module area).
95 * Therefore the new displacement will always fit.
97 disp = *(s32 *)&p->ainsn.insn[1];
98 addr = (u64)(unsigned long)p->addr;
99 new_addr = (u64)(unsigned long)p->ainsn.insn;
100 new_disp = ((addr + (disp * 2)) - new_addr) / 2;
101 *(s32 *)&p->ainsn.insn[1] = new_disp;
103 NOKPROBE_SYMBOL(copy_instruction);
105 static inline int is_kernel_addr(void *addr)
107 return addr < (void *)_end;
110 static int s390_get_insn_slot(struct kprobe *p)
113 * Get an insn slot that is within the same 2GB area like the original
114 * instruction. That way instructions with a 32bit signed displacement
115 * field can be patched and executed within the insn slot.
117 p->ainsn.insn = NULL;
118 if (is_kernel_addr(p->addr))
119 p->ainsn.insn = get_dmainsn_slot();
120 else if (is_module_addr(p->addr))
121 p->ainsn.insn = get_insn_slot();
122 return p->ainsn.insn ? 0 : -ENOMEM;
124 NOKPROBE_SYMBOL(s390_get_insn_slot);
126 static void s390_free_insn_slot(struct kprobe *p)
130 if (is_kernel_addr(p->addr))
131 free_dmainsn_slot(p->ainsn.insn, 0);
133 free_insn_slot(p->ainsn.insn, 0);
134 p->ainsn.insn = NULL;
136 NOKPROBE_SYMBOL(s390_free_insn_slot);
138 int arch_prepare_kprobe(struct kprobe *p)
140 if ((unsigned long) p->addr & 0x01)
142 /* Make sure the probe isn't going on a difficult instruction */
143 if (probe_is_prohibited_opcode(p->addr))
145 if (s390_get_insn_slot(p))
150 NOKPROBE_SYMBOL(arch_prepare_kprobe);
152 int arch_check_ftrace_location(struct kprobe *p)
157 struct swap_insn_args {
159 unsigned int arm_kprobe : 1;
162 static int swap_instruction(void *data)
164 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
165 unsigned long status = kcb->kprobe_status;
166 struct swap_insn_args *args = data;
167 struct ftrace_insn new_insn, *insn;
168 struct kprobe *p = args->p;
171 new_insn.opc = args->arm_kprobe ? BREAKPOINT_INSTRUCTION : p->opcode;
172 len = sizeof(new_insn.opc);
173 if (!p->ainsn.is_ftrace_insn)
175 len = sizeof(new_insn);
176 insn = (struct ftrace_insn *) p->addr;
177 if (args->arm_kprobe) {
178 if (is_ftrace_nop(insn))
179 new_insn.disp = KPROBE_ON_FTRACE_NOP;
181 new_insn.disp = KPROBE_ON_FTRACE_CALL;
183 ftrace_generate_call_insn(&new_insn, (unsigned long)p->addr);
184 if (insn->disp == KPROBE_ON_FTRACE_NOP)
185 ftrace_generate_nop_insn(&new_insn);
188 kcb->kprobe_status = KPROBE_SWAP_INST;
189 s390_kernel_write(p->addr, &new_insn, len);
190 kcb->kprobe_status = status;
193 NOKPROBE_SYMBOL(swap_instruction);
195 void arch_arm_kprobe(struct kprobe *p)
197 struct swap_insn_args args = {.p = p, .arm_kprobe = 1};
199 stop_machine_cpuslocked(swap_instruction, &args, NULL);
201 NOKPROBE_SYMBOL(arch_arm_kprobe);
203 void arch_disarm_kprobe(struct kprobe *p)
205 struct swap_insn_args args = {.p = p, .arm_kprobe = 0};
207 stop_machine_cpuslocked(swap_instruction, &args, NULL);
209 NOKPROBE_SYMBOL(arch_disarm_kprobe);
211 void arch_remove_kprobe(struct kprobe *p)
213 s390_free_insn_slot(p);
215 NOKPROBE_SYMBOL(arch_remove_kprobe);
217 static void enable_singlestep(struct kprobe_ctlblk *kcb,
218 struct pt_regs *regs,
221 struct per_regs per_kprobe;
223 /* Set up the PER control registers %cr9-%cr11 */
224 per_kprobe.control = PER_EVENT_IFETCH;
225 per_kprobe.start = ip;
228 /* Save control regs and psw mask */
229 __ctl_store(kcb->kprobe_saved_ctl, 9, 11);
230 kcb->kprobe_saved_imask = regs->psw.mask &
231 (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT);
233 /* Set PER control regs, turns on single step for the given address */
234 __ctl_load(per_kprobe, 9, 11);
235 regs->psw.mask |= PSW_MASK_PER;
236 regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT);
239 NOKPROBE_SYMBOL(enable_singlestep);
241 static void disable_singlestep(struct kprobe_ctlblk *kcb,
242 struct pt_regs *regs,
245 /* Restore control regs and psw mask, set new psw address */
246 __ctl_load(kcb->kprobe_saved_ctl, 9, 11);
247 regs->psw.mask &= ~PSW_MASK_PER;
248 regs->psw.mask |= kcb->kprobe_saved_imask;
251 NOKPROBE_SYMBOL(disable_singlestep);
254 * Activate a kprobe by storing its pointer to current_kprobe. The
255 * previous kprobe is stored in kcb->prev_kprobe. A stack of up to
256 * two kprobes can be active, see KPROBE_REENTER.
258 static void push_kprobe(struct kprobe_ctlblk *kcb, struct kprobe *p)
260 kcb->prev_kprobe.kp = __this_cpu_read(current_kprobe);
261 kcb->prev_kprobe.status = kcb->kprobe_status;
262 __this_cpu_write(current_kprobe, p);
264 NOKPROBE_SYMBOL(push_kprobe);
267 * Deactivate a kprobe by backing up to the previous state. If the
268 * current state is KPROBE_REENTER prev_kprobe.kp will be non-NULL,
269 * for any other state prev_kprobe.kp will be NULL.
271 static void pop_kprobe(struct kprobe_ctlblk *kcb)
273 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
274 kcb->kprobe_status = kcb->prev_kprobe.status;
275 kcb->prev_kprobe.kp = NULL;
277 NOKPROBE_SYMBOL(pop_kprobe);
279 void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
281 ri->ret_addr = (kprobe_opcode_t *) regs->gprs[14];
283 /* Replace the return addr with trampoline addr */
284 regs->gprs[14] = (unsigned long) &kretprobe_trampoline;
286 NOKPROBE_SYMBOL(arch_prepare_kretprobe);
288 static void kprobe_reenter_check(struct kprobe_ctlblk *kcb, struct kprobe *p)
290 switch (kcb->kprobe_status) {
291 case KPROBE_HIT_SSDONE:
292 case KPROBE_HIT_ACTIVE:
293 kprobes_inc_nmissed_count(p);
299 * A kprobe on the code path to single step an instruction
300 * is a BUG. The code path resides in the .kprobes.text
301 * section and is executed with interrupts disabled.
303 printk(KERN_EMERG "Invalid kprobe detected at %p.\n", p->addr);
308 NOKPROBE_SYMBOL(kprobe_reenter_check);
310 static int kprobe_handler(struct pt_regs *regs)
312 struct kprobe_ctlblk *kcb;
316 * We want to disable preemption for the entire duration of kprobe
317 * processing. That includes the calls to the pre/post handlers
318 * and single stepping the kprobe instruction.
321 kcb = get_kprobe_ctlblk();
322 p = get_kprobe((void *)(regs->psw.addr - 2));
325 if (kprobe_running()) {
327 * We have hit a kprobe while another is still
328 * active. This can happen in the pre and post
329 * handler. Single step the instruction of the
330 * new probe but do not call any handler function
331 * of this secondary kprobe.
332 * push_kprobe and pop_kprobe saves and restores
333 * the currently active kprobe.
335 kprobe_reenter_check(kcb, p);
337 kcb->kprobe_status = KPROBE_REENTER;
340 * If we have no pre-handler or it returned 0, we
341 * continue with single stepping. If we have a
342 * pre-handler and it returned non-zero, it prepped
343 * for calling the break_handler below on re-entry
344 * for jprobe processing, so get out doing nothing
348 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
349 if (p->pre_handler && p->pre_handler(p, regs))
351 kcb->kprobe_status = KPROBE_HIT_SS;
353 enable_singlestep(kcb, regs, (unsigned long) p->ainsn.insn);
355 } else if (kprobe_running()) {
356 p = __this_cpu_read(current_kprobe);
357 if (p->break_handler && p->break_handler(p, regs)) {
359 * Continuation after the jprobe completed and
360 * caused the jprobe_return trap. The jprobe
361 * break_handler "returns" to the original
362 * function that still has the kprobe breakpoint
363 * installed. We continue with single stepping.
365 kcb->kprobe_status = KPROBE_HIT_SS;
366 enable_singlestep(kcb, regs,
367 (unsigned long) p->ainsn.insn);
370 * No kprobe at this address and the current kprobe
371 * has no break handler (no jprobe!). The kernel just
372 * exploded, let the standard trap handler pick up the
376 * No kprobe at this address and no active kprobe. The trap has
377 * not been caused by a kprobe breakpoint. The race of breakpoint
378 * vs. kprobe remove does not exist because on s390 as we use
379 * stop_machine to arm/disarm the breakpoints.
381 preempt_enable_no_resched();
384 NOKPROBE_SYMBOL(kprobe_handler);
387 * Function return probe trampoline:
388 * - init_kprobes() establishes a probepoint here
389 * - When the probed function returns, this probe
390 * causes the handlers to fire
392 static void __used kretprobe_trampoline_holder(void)
394 asm volatile(".global kretprobe_trampoline\n"
395 "kretprobe_trampoline: bcr 0,0\n");
399 * Called when the probe at kretprobe trampoline is hit
401 static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
403 struct kretprobe_instance *ri;
404 struct hlist_head *head, empty_rp;
405 struct hlist_node *tmp;
406 unsigned long flags, orig_ret_address;
407 unsigned long trampoline_address;
408 kprobe_opcode_t *correct_ret_addr;
410 INIT_HLIST_HEAD(&empty_rp);
411 kretprobe_hash_lock(current, &head, &flags);
414 * It is possible to have multiple instances associated with a given
415 * task either because an multiple functions in the call path
416 * have a return probe installed on them, and/or more than one return
417 * return probe was registered for a target function.
419 * We can handle this because:
420 * - instances are always inserted at the head of the list
421 * - when multiple return probes are registered for the same
422 * function, the first instance's ret_addr will point to the
423 * real return address, and all the rest will point to
424 * kretprobe_trampoline
427 orig_ret_address = 0;
428 correct_ret_addr = NULL;
429 trampoline_address = (unsigned long) &kretprobe_trampoline;
430 hlist_for_each_entry_safe(ri, tmp, head, hlist) {
431 if (ri->task != current)
432 /* another task is sharing our hash bucket */
435 orig_ret_address = (unsigned long) ri->ret_addr;
437 if (orig_ret_address != trampoline_address)
439 * This is the real return address. Any other
440 * instances associated with this task are for
441 * other calls deeper on the call stack
446 kretprobe_assert(ri, orig_ret_address, trampoline_address);
448 correct_ret_addr = ri->ret_addr;
449 hlist_for_each_entry_safe(ri, tmp, head, hlist) {
450 if (ri->task != current)
451 /* another task is sharing our hash bucket */
454 orig_ret_address = (unsigned long) ri->ret_addr;
456 if (ri->rp && ri->rp->handler) {
457 ri->ret_addr = correct_ret_addr;
458 ri->rp->handler(ri, regs);
461 recycle_rp_inst(ri, &empty_rp);
463 if (orig_ret_address != trampoline_address)
465 * This is the real return address. Any other
466 * instances associated with this task are for
467 * other calls deeper on the call stack
472 regs->psw.addr = orig_ret_address;
474 pop_kprobe(get_kprobe_ctlblk());
475 kretprobe_hash_unlock(current, &flags);
476 preempt_enable_no_resched();
478 hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
479 hlist_del(&ri->hlist);
483 * By returning a non-zero value, we are telling
484 * kprobe_handler() that we don't want the post_handler
485 * to run (and have re-enabled preemption)
489 NOKPROBE_SYMBOL(trampoline_probe_handler);
492 * Called after single-stepping. p->addr is the address of the
493 * instruction whose first byte has been replaced by the "breakpoint"
494 * instruction. To avoid the SMP problems that can occur when we
495 * temporarily put back the original opcode to single-step, we
496 * single-stepped a copy of the instruction. The address of this
497 * copy is p->ainsn.insn.
499 static void resume_execution(struct kprobe *p, struct pt_regs *regs)
501 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
502 unsigned long ip = regs->psw.addr;
503 int fixup = probe_get_fixup_type(p->ainsn.insn);
505 /* Check if the kprobes location is an enabled ftrace caller */
506 if (p->ainsn.is_ftrace_insn) {
507 struct ftrace_insn *insn = (struct ftrace_insn *) p->addr;
508 struct ftrace_insn call_insn;
510 ftrace_generate_call_insn(&call_insn, (unsigned long) p->addr);
512 * A kprobe on an enabled ftrace call site actually single
513 * stepped an unconditional branch (ftrace nop equivalent).
514 * Now we need to fixup things and pretend that a brasl r0,...
515 * was executed instead.
517 if (insn->disp == KPROBE_ON_FTRACE_CALL) {
518 ip += call_insn.disp * 2 - MCOUNT_INSN_SIZE;
519 regs->gprs[0] = (unsigned long)p->addr + sizeof(*insn);
523 if (fixup & FIXUP_PSW_NORMAL)
524 ip += (unsigned long) p->addr - (unsigned long) p->ainsn.insn;
526 if (fixup & FIXUP_BRANCH_NOT_TAKEN) {
527 int ilen = insn_length(p->ainsn.insn[0] >> 8);
528 if (ip - (unsigned long) p->ainsn.insn == ilen)
529 ip = (unsigned long) p->addr + ilen;
532 if (fixup & FIXUP_RETURN_REGISTER) {
533 int reg = (p->ainsn.insn[0] & 0xf0) >> 4;
534 regs->gprs[reg] += (unsigned long) p->addr -
535 (unsigned long) p->ainsn.insn;
538 disable_singlestep(kcb, regs, ip);
540 NOKPROBE_SYMBOL(resume_execution);
542 static int post_kprobe_handler(struct pt_regs *regs)
544 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
545 struct kprobe *p = kprobe_running();
550 resume_execution(p, regs);
551 if (kcb->kprobe_status != KPROBE_REENTER && p->post_handler) {
552 kcb->kprobe_status = KPROBE_HIT_SSDONE;
553 p->post_handler(p, regs, 0);
556 preempt_enable_no_resched();
559 * if somebody else is singlestepping across a probe point, psw mask
560 * will have PER set, in which case, continue the remaining processing
561 * of do_single_step, as if this is not a probe hit.
563 if (regs->psw.mask & PSW_MASK_PER)
568 NOKPROBE_SYMBOL(post_kprobe_handler);
570 static int kprobe_trap_handler(struct pt_regs *regs, int trapnr)
572 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
573 struct kprobe *p = kprobe_running();
574 const struct exception_table_entry *entry;
576 switch(kcb->kprobe_status) {
577 case KPROBE_SWAP_INST:
578 /* We are here because the instruction replacement failed */
583 * We are here because the instruction being single
584 * stepped caused a page fault. We reset the current
585 * kprobe and the nip points back to the probe address
586 * and allow the page fault handler to continue as a
589 disable_singlestep(kcb, regs, (unsigned long) p->addr);
591 preempt_enable_no_resched();
593 case KPROBE_HIT_ACTIVE:
594 case KPROBE_HIT_SSDONE:
596 * We increment the nmissed count for accounting,
597 * we can also use npre/npostfault count for accounting
598 * these specific fault cases.
600 kprobes_inc_nmissed_count(p);
603 * We come here because instructions in the pre/post
604 * handler caused the page_fault, this could happen
605 * if handler tries to access user space by
606 * copy_from_user(), get_user() etc. Let the
607 * user-specified handler try to fix it first.
609 if (p->fault_handler && p->fault_handler(p, regs, trapnr))
613 * In case the user-specified fault handler returned
614 * zero, try to fix up.
616 entry = search_exception_tables(regs->psw.addr);
618 regs->psw.addr = extable_fixup(entry);
623 * fixup_exception() could not handle it,
624 * Let do_page_fault() fix it.
632 NOKPROBE_SYMBOL(kprobe_trap_handler);
634 int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
638 if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
640 ret = kprobe_trap_handler(regs, trapnr);
641 if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
642 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
645 NOKPROBE_SYMBOL(kprobe_fault_handler);
648 * Wrapper routine to for handling exceptions.
650 int kprobe_exceptions_notify(struct notifier_block *self,
651 unsigned long val, void *data)
653 struct die_args *args = (struct die_args *) data;
654 struct pt_regs *regs = args->regs;
655 int ret = NOTIFY_DONE;
657 if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
662 if (kprobe_handler(regs))
666 if (post_kprobe_handler(regs))
670 if (!preemptible() && kprobe_running() &&
671 kprobe_trap_handler(regs, args->trapnr))
678 if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
679 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
683 NOKPROBE_SYMBOL(kprobe_exceptions_notify);
685 int setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
687 struct jprobe *jp = container_of(p, struct jprobe, kp);
688 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
691 memcpy(&kcb->jprobe_saved_regs, regs, sizeof(struct pt_regs));
693 /* setup return addr to the jprobe handler routine */
694 regs->psw.addr = (unsigned long) jp->entry;
695 regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT);
697 /* r15 is the stack pointer */
698 stack = (unsigned long) regs->gprs[15];
700 memcpy(kcb->jprobes_stack, (void *) stack, MIN_STACK_SIZE(stack));
703 * jprobes use jprobe_return() which skips the normal return
704 * path of the function, and this messes up the accounting of the
705 * function graph tracer to get messed up.
707 * Pause function graph tracing while performing the jprobe function.
709 pause_graph_tracing();
712 NOKPROBE_SYMBOL(setjmp_pre_handler);
714 void jprobe_return(void)
716 asm volatile(".word 0x0002");
718 NOKPROBE_SYMBOL(jprobe_return);
720 int longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
722 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
725 /* It's OK to start function graph tracing again */
726 unpause_graph_tracing();
728 stack = (unsigned long) kcb->jprobe_saved_regs.gprs[15];
730 /* Put the regs back */
731 memcpy(regs, &kcb->jprobe_saved_regs, sizeof(struct pt_regs));
732 /* put the stack back */
733 memcpy((void *) stack, kcb->jprobes_stack, MIN_STACK_SIZE(stack));
734 preempt_enable_no_resched();
737 NOKPROBE_SYMBOL(longjmp_break_handler);
739 static struct kprobe trampoline = {
740 .addr = (kprobe_opcode_t *) &kretprobe_trampoline,
741 .pre_handler = trampoline_probe_handler
744 int __init arch_init_kprobes(void)
746 return register_kprobe(&trampoline);
749 int arch_trampoline_kprobe(struct kprobe *p)
751 return p->addr == (kprobe_opcode_t *) &kretprobe_trampoline;
753 NOKPROBE_SYMBOL(arch_trampoline_kprobe);