2 * arch/arm64/kernel/ftrace.c
4 * Copyright (C) 2013 Linaro Limited
5 * Author: AKASHI Takahiro <takahiro.akashi@linaro.org>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
12 #include <linux/ftrace.h>
13 #include <linux/module.h>
14 #include <linux/swab.h>
15 #include <linux/uaccess.h>
17 #include <asm/cacheflush.h>
18 #include <asm/debug-monitors.h>
19 #include <asm/ftrace.h>
22 #ifdef CONFIG_DYNAMIC_FTRACE
24 * Replace a single instruction, which may be a branch or NOP.
25 * If @validate == true, a replaced instruction is checked against 'old'.
27 static int ftrace_modify_code(unsigned long pc, u32 old, u32 new,
34 * We are paranoid about modifying text, as if a bug were to happen, it
35 * could cause us to read or write to someplace that could cause harm.
36 * Carefully read and modify the code with aarch64_insn_*() which uses
37 * probe_kernel_*(), and make sure what we read is what we expected it
38 * to be before modifying it.
41 if (aarch64_insn_read((void *)pc, &replaced))
47 if (aarch64_insn_patch_text_nosync((void *)pc, new))
54 * Replace tracer function in ftrace_caller()
56 int ftrace_update_ftrace_func(ftrace_func_t func)
61 pc = (unsigned long)&ftrace_call;
62 new = aarch64_insn_gen_branch_imm(pc, (unsigned long)func,
63 AARCH64_INSN_BRANCH_LINK);
65 return ftrace_modify_code(pc, 0, new, false);
69 * Turn on the call to ftrace_caller() in instrumented function
71 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
73 unsigned long pc = rec->ip;
75 long offset = (long)addr - (long)pc;
77 if (offset < -SZ_128M || offset >= SZ_128M) {
78 #ifdef CONFIG_ARM64_MODULE_PLTS
79 struct plt_entry trampoline, *dst;
83 * On kernels that support module PLTs, the offset between the
84 * branch instruction and its target may legally exceed the
85 * range of an ordinary relative 'bl' opcode. In this case, we
86 * need to branch via a trampoline in the module.
88 * NOTE: __module_text_address() must be called with preemption
89 * disabled, but we can rely on ftrace_lock to ensure that 'mod'
90 * retains its validity throughout the remainder of this code.
93 mod = __module_text_address(pc);
100 * There is only one ftrace trampoline per module. For now,
101 * this is not a problem since on arm64, all dynamic ftrace
102 * invocations are routed via ftrace_caller(). This will need
103 * to be revisited if support for multiple ftrace entry points
104 * is added in the future, but for now, the pr_err() below
105 * deals with a theoretical issue only.
107 dst = mod->arch.ftrace_trampoline;
108 trampoline = get_plt_entry(addr);
109 if (!plt_entries_equal(dst, &trampoline)) {
110 if (!plt_entries_equal(dst, &(struct plt_entry){})) {
111 pr_err("ftrace: far branches to multiple entry points unsupported inside a single module\n");
115 /* point the trampoline to our ftrace entry point */
116 module_disable_ro(mod);
118 module_enable_ro(mod, true);
121 * Ensure updated trampoline is visible to instruction
122 * fetch before we patch in the branch. Although the
123 * architecture doesn't require an IPI in this case,
124 * Neoverse-N1 erratum #1542419 does require one
125 * if the TLB maintenance in module_enable_ro() is
126 * skipped due to rodata_enabled. It doesn't seem worth
127 * it to make it conditional given that this is
128 * certainly not a fast-path.
130 flush_icache_range((unsigned long)&dst[0],
131 (unsigned long)&dst[1]);
133 addr = (unsigned long)dst;
134 #else /* CONFIG_ARM64_MODULE_PLTS */
136 #endif /* CONFIG_ARM64_MODULE_PLTS */
139 old = aarch64_insn_gen_nop();
140 new = aarch64_insn_gen_branch_imm(pc, addr, AARCH64_INSN_BRANCH_LINK);
142 return ftrace_modify_code(pc, old, new, true);
146 * Turn off the call to ftrace_caller() in instrumented function
148 int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec,
151 unsigned long pc = rec->ip;
152 bool validate = true;
154 long offset = (long)addr - (long)pc;
156 if (offset < -SZ_128M || offset >= SZ_128M) {
157 #ifdef CONFIG_ARM64_MODULE_PLTS
161 * 'mod' is only set at module load time, but if we end up
162 * dealing with an out-of-range condition, we can assume it
163 * is due to a module being loaded far away from the kernel.
167 mod = __module_text_address(pc);
175 * The instruction we are about to patch may be a branch and
176 * link instruction that was redirected via a PLT entry. In
177 * this case, the normal validation will fail, but we can at
178 * least check that we are dealing with a branch and link
179 * instruction that points into the right module.
181 if (aarch64_insn_read((void *)pc, &replaced))
184 if (!aarch64_insn_is_bl(replaced) ||
185 !within_module(pc + aarch64_get_branch_offset(replaced),
190 #else /* CONFIG_ARM64_MODULE_PLTS */
192 #endif /* CONFIG_ARM64_MODULE_PLTS */
194 old = aarch64_insn_gen_branch_imm(pc, addr,
195 AARCH64_INSN_BRANCH_LINK);
198 new = aarch64_insn_gen_nop();
200 return ftrace_modify_code(pc, old, new, validate);
203 void arch_ftrace_update_code(int command)
205 ftrace_modify_all_code(command);
208 int __init ftrace_dyn_arch_init(void)
212 #endif /* CONFIG_DYNAMIC_FTRACE */
214 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
216 * function_graph tracer expects ftrace_return_to_handler() to be called
217 * on the way back to parent. For this purpose, this function is called
218 * in _mcount() or ftrace_caller() to replace return address (*parent) on
219 * the call stack to return_to_handler.
221 * Note that @frame_pointer is used only for sanity check later.
223 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
224 unsigned long frame_pointer)
226 unsigned long return_hooker = (unsigned long)&return_to_handler;
228 struct ftrace_graph_ent trace;
231 if (unlikely(atomic_read(¤t->tracing_graph_pause)))
236 * No protection against faulting at *parent, which may be seen
237 * on other archs. It's unlikely on AArch64.
241 trace.func = self_addr;
242 trace.depth = current->curr_ret_stack + 1;
244 /* Only trace if the calling function expects to */
245 if (!ftrace_graph_entry(&trace))
248 err = ftrace_push_return_trace(old, self_addr, &trace.depth,
249 frame_pointer, NULL);
253 *parent = return_hooker;
256 #ifdef CONFIG_DYNAMIC_FTRACE
258 * Turn on/off the call to ftrace_graph_caller() in ftrace_caller()
259 * depending on @enable.
261 static int ftrace_modify_graph_caller(bool enable)
263 unsigned long pc = (unsigned long)&ftrace_graph_call;
266 branch = aarch64_insn_gen_branch_imm(pc,
267 (unsigned long)ftrace_graph_caller,
268 AARCH64_INSN_BRANCH_NOLINK);
269 nop = aarch64_insn_gen_nop();
272 return ftrace_modify_code(pc, nop, branch, true);
274 return ftrace_modify_code(pc, branch, nop, true);
277 int ftrace_enable_ftrace_graph_caller(void)
279 return ftrace_modify_graph_caller(true);
282 int ftrace_disable_ftrace_graph_caller(void)
284 return ftrace_modify_graph_caller(false);
286 #endif /* CONFIG_DYNAMIC_FTRACE */
287 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */