2 * arch/arm64/kernel/entry-ftrace.S
4 * Copyright (C) 2013 Linaro Limited
5 * Author: AKASHI Takahiro <takahiro.akashi@linaro.org>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
12 #include <linux/linkage.h>
13 #include <asm/ftrace.h>
17 * Gcc with -pg will put the following code in the beginning of each function:
20 * [function's body ...]
21 * "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic
24 * Please note that x0 as an argument will not be used here because we can
25 * get lr(x30) of instrumented function at any time by winding up call stack
26 * as long as the kernel is compiled without -fomit-frame-pointer.
27 * (or CONFIG_FRAME_POINTER, this is forced on arm64)
29 * stack layout after mcount_enter in _mcount():
31 * current sp/fp => 0:+-----+
32 * in _mcount() | x29 | -> instrumented function's fp
34 * | x30 | -> _mcount()'s lr (= instrumented function's pc)
35 * old sp => +16:+-----+
36 * when instrumented | |
37 * function calls | ... |
40 * instrumented => +xx:+-----+
41 * function's fp | x29 | -> parent's fp
43 * | x30 | -> instrumented function's lr (= parent's pc)
49 stp x29, x30, [sp, #-16]!
54 ldp x29, x30, [sp], #16
58 .macro mcount_adjust_addr rd, rn
59 sub \rd, \rn, #AARCH64_INSN_SIZE
62 /* for instrumented function's parent */
63 .macro mcount_get_parent_fp reg
68 /* for instrumented function */
69 .macro mcount_get_pc0 reg
70 mcount_adjust_addr \reg, x30
73 .macro mcount_get_pc reg
75 mcount_adjust_addr \reg, \reg
78 .macro mcount_get_lr reg
83 .macro mcount_get_lr_addr reg
88 #ifndef CONFIG_DYNAMIC_FTRACE
90 * void _mcount(unsigned long return_address)
91 * @return_address: return address to instrumented function
93 * This function makes calls, if enabled, to:
94 * - tracer function to probe instrumented function's entry,
95 * - ftrace_graph_caller to set up an exit hook
100 adrp x0, ftrace_trace_function
101 ldr x2, [x0, #:lo12:ftrace_trace_function]
103 cmp x0, x2 // if (ftrace_trace_function
104 b.eq skip_ftrace_call // != ftrace_stub) {
106 mcount_get_pc x0 // function's pc
107 mcount_get_lr x1 // function's lr (= parent's pc)
108 blr x2 // (*ftrace_trace_function)(pc, lr);
110 #ifndef CONFIG_FUNCTION_GRAPH_TRACER
111 skip_ftrace_call: // return;
114 mcount_exit // return;
117 adrp x1, ftrace_graph_return
118 ldr x2, [x1, #:lo12:ftrace_graph_return]
119 cmp x0, x2 // if ((ftrace_graph_return
120 b.ne ftrace_graph_caller // != ftrace_stub)
122 adrp x1, ftrace_graph_entry // || (ftrace_graph_entry
123 adrp x0, ftrace_graph_entry_stub // != ftrace_graph_entry_stub))
124 ldr x2, [x1, #:lo12:ftrace_graph_entry]
125 add x0, x0, #:lo12:ftrace_graph_entry_stub
127 b.ne ftrace_graph_caller // ftrace_graph_caller();
130 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
133 #else /* CONFIG_DYNAMIC_FTRACE */
135 * _mcount() is used to build the kernel with -pg option, but all the branch
136 * instructions to _mcount() are replaced to NOP initially at kernel start up,
137 * and later on, NOP to branch to ftrace_caller() when enabled or branch to
138 * NOP when disabled per-function base.
145 * void ftrace_caller(unsigned long return_address)
146 * @return_address: return address to instrumented function
148 * This function is a counterpart of _mcount() in 'static' ftrace, and
150 * - tracer function to probe instrumented function's entry,
151 * - ftrace_graph_caller to set up an exit hook
156 mcount_get_pc0 x0 // function's pc
157 mcount_get_lr x1 // function's lr
160 ftrace_call: // tracer(pc, lr);
161 nop // This will be replaced with "bl xxx"
162 // where xxx can be any kind of tracer.
164 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
165 .global ftrace_graph_call
166 ftrace_graph_call: // ftrace_graph_caller();
167 nop // If enabled, this will be replaced
168 // "b ftrace_graph_caller"
172 ENDPROC(ftrace_caller)
173 #endif /* CONFIG_DYNAMIC_FTRACE */
179 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
180 /* save return value regs*/
181 .macro save_return_regs
184 stp x2, x3, [sp, #16]
185 stp x4, x5, [sp, #32]
186 stp x6, x7, [sp, #48]
189 /* restore return value regs*/
190 .macro restore_return_regs
192 ldp x2, x3, [sp, #16]
193 ldp x4, x5, [sp, #32]
194 ldp x6, x7, [sp, #48]
199 * void ftrace_graph_caller(void)
201 * Called from _mcount() or ftrace_caller() when function_graph tracer is
203 * This function w/ prepare_ftrace_return() fakes link register's value on
204 * the call stack in order to intercept instrumented function's return path
205 * and run return_to_handler() later on its exit.
207 ENTRY(ftrace_graph_caller)
208 mcount_get_lr_addr x0 // pointer to function's saved lr
209 mcount_get_pc x1 // function's pc
210 mcount_get_parent_fp x2 // parent's fp
211 bl prepare_ftrace_return // prepare_ftrace_return(&lr, pc, fp)
214 ENDPROC(ftrace_graph_caller)
217 * void return_to_handler(void)
219 * Run ftrace_return_to_handler() before going back to parent.
220 * @fp is checked against the value passed by ftrace_graph_caller()
221 * only when HAVE_FUNCTION_GRAPH_FP_TEST is enabled.
223 ENTRY(return_to_handler)
225 mov x0, x29 // parent's fp
226 bl ftrace_return_to_handler// addr = ftrace_return_to_hander(fp);
227 mov x30, x0 // restore the original return address
230 END(return_to_handler)
231 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */