GNU Linux-libre 4.19.264-gnu1
[releases.git] / arch / arm / kernel / ftrace.c
1 /*
2  * Dynamic function tracing support.
3  *
4  * Copyright (C) 2008 Abhishek Sagar <sagar.abhishek@gmail.com>
5  * Copyright (C) 2010 Rabin Vincent <rabin@rab.in>
6  *
7  * For licencing details, see COPYING.
8  *
9  * Defines low-level handling of mcount calls when the kernel
10  * is compiled with the -pg flag. When using dynamic ftrace, the
11  * mcount call-sites get patched with NOP till they are enabled.
12  * All code mutation routines here are called under stop_machine().
13  */
14
15 #include <linux/ftrace.h>
16 #include <linux/uaccess.h>
17 #include <linux/module.h>
18 #include <linux/stop_machine.h>
19
20 #include <asm/cacheflush.h>
21 #include <asm/opcodes.h>
22 #include <asm/ftrace.h>
23 #include <asm/insn.h>
24 #include <asm/set_memory.h>
25
26 #ifdef CONFIG_THUMB2_KERNEL
27 #define NOP             0xf85deb04      /* pop.w {lr} */
28 #else
29 #define NOP             0xe8bd4000      /* pop {lr} */
30 #endif
31
32 #ifdef CONFIG_DYNAMIC_FTRACE
33
34 static int __ftrace_modify_code(void *data)
35 {
36         int *command = data;
37
38         set_kernel_text_rw();
39         ftrace_modify_all_code(*command);
40         set_kernel_text_ro();
41
42         return 0;
43 }
44
45 void arch_ftrace_update_code(int command)
46 {
47         stop_machine(__ftrace_modify_code, &command, NULL);
48 }
49
50 #ifdef CONFIG_OLD_MCOUNT
51 #define OLD_MCOUNT_ADDR ((unsigned long) mcount)
52 #define OLD_FTRACE_ADDR ((unsigned long) ftrace_caller_old)
53
54 #define OLD_NOP         0xe1a00000      /* mov r0, r0 */
55
56 static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec)
57 {
58         return rec->arch.old_mcount ? OLD_NOP : NOP;
59 }
60
61 static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr)
62 {
63         if (!rec->arch.old_mcount)
64                 return addr;
65
66         if (addr == MCOUNT_ADDR)
67                 addr = OLD_MCOUNT_ADDR;
68         else if (addr == FTRACE_ADDR)
69                 addr = OLD_FTRACE_ADDR;
70
71         return addr;
72 }
73 #else
74 static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec)
75 {
76         return NOP;
77 }
78
79 static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr)
80 {
81         return addr;
82 }
83 #endif
84
85 int ftrace_arch_code_modify_prepare(void)
86 {
87         set_all_modules_text_rw();
88         return 0;
89 }
90
91 int ftrace_arch_code_modify_post_process(void)
92 {
93         set_all_modules_text_ro();
94         /* Make sure any TLB misses during machine stop are cleared. */
95         flush_tlb_all();
96         return 0;
97 }
98
99 static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr,
100                                          bool warn)
101 {
102         return arm_gen_branch_link(pc, addr, warn);
103 }
104
105 static int ftrace_modify_code(unsigned long pc, unsigned long old,
106                               unsigned long new, bool validate)
107 {
108         unsigned long replaced;
109
110         if (IS_ENABLED(CONFIG_THUMB2_KERNEL)) {
111                 old = __opcode_to_mem_thumb32(old);
112                 new = __opcode_to_mem_thumb32(new);
113         } else {
114                 old = __opcode_to_mem_arm(old);
115                 new = __opcode_to_mem_arm(new);
116         }
117
118         if (validate) {
119                 if (probe_kernel_read(&replaced, (void *)pc, MCOUNT_INSN_SIZE))
120                         return -EFAULT;
121
122                 if (replaced != old)
123                         return -EINVAL;
124         }
125
126         if (probe_kernel_write((void *)pc, &new, MCOUNT_INSN_SIZE))
127                 return -EPERM;
128
129         flush_icache_range(pc, pc + MCOUNT_INSN_SIZE);
130
131         return 0;
132 }
133
134 int ftrace_update_ftrace_func(ftrace_func_t func)
135 {
136         unsigned long pc;
137         unsigned long new;
138         int ret;
139
140         pc = (unsigned long)&ftrace_call;
141         new = ftrace_call_replace(pc, (unsigned long)func, true);
142
143         ret = ftrace_modify_code(pc, 0, new, false);
144
145 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
146         if (!ret) {
147                 pc = (unsigned long)&ftrace_regs_call;
148                 new = ftrace_call_replace(pc, (unsigned long)func, true);
149
150                 ret = ftrace_modify_code(pc, 0, new, false);
151         }
152 #endif
153
154 #ifdef CONFIG_OLD_MCOUNT
155         if (!ret) {
156                 pc = (unsigned long)&ftrace_call_old;
157                 new = ftrace_call_replace(pc, (unsigned long)func, true);
158
159                 ret = ftrace_modify_code(pc, 0, new, false);
160         }
161 #endif
162
163         return ret;
164 }
165
166 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
167 {
168         unsigned long new, old;
169         unsigned long ip = rec->ip;
170         unsigned long aaddr = adjust_address(rec, addr);
171         struct module *mod = NULL;
172
173 #ifdef CONFIG_ARM_MODULE_PLTS
174         mod = rec->arch.mod;
175 #endif
176
177         old = ftrace_nop_replace(rec);
178
179         new = ftrace_call_replace(ip, aaddr, !mod);
180 #ifdef CONFIG_ARM_MODULE_PLTS
181         if (!new && mod) {
182                 aaddr = get_module_plt(mod, ip, aaddr);
183                 new = ftrace_call_replace(ip, aaddr, true);
184         }
185 #endif
186
187         return ftrace_modify_code(rec->ip, old, new, true);
188 }
189
190 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
191
192 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
193                                 unsigned long addr)
194 {
195         unsigned long new, old;
196         unsigned long ip = rec->ip;
197
198         old = ftrace_call_replace(ip, adjust_address(rec, old_addr), true);
199
200         new = ftrace_call_replace(ip, adjust_address(rec, addr), true);
201
202         return ftrace_modify_code(rec->ip, old, new, true);
203 }
204
205 #endif
206
207 int ftrace_make_nop(struct module *mod,
208                     struct dyn_ftrace *rec, unsigned long addr)
209 {
210         unsigned long aaddr = adjust_address(rec, addr);
211         unsigned long ip = rec->ip;
212         unsigned long old;
213         unsigned long new;
214         int ret;
215
216 #ifdef CONFIG_ARM_MODULE_PLTS
217         /* mod is only supplied during module loading */
218         if (!mod)
219                 mod = rec->arch.mod;
220         else
221                 rec->arch.mod = mod;
222 #endif
223
224         old = ftrace_call_replace(ip, aaddr,
225                                   !IS_ENABLED(CONFIG_ARM_MODULE_PLTS) || !mod);
226 #ifdef CONFIG_ARM_MODULE_PLTS
227         if (!old && mod) {
228                 aaddr = get_module_plt(mod, ip, aaddr);
229                 old = ftrace_call_replace(ip, aaddr, true);
230         }
231 #endif
232
233         new = ftrace_nop_replace(rec);
234         ret = ftrace_modify_code(ip, old, new, true);
235
236 #ifdef CONFIG_OLD_MCOUNT
237         if (ret == -EINVAL && addr == MCOUNT_ADDR) {
238                 rec->arch.old_mcount = true;
239
240                 old = ftrace_call_replace(ip, adjust_address(rec, addr), true);
241                 new = ftrace_nop_replace(rec);
242                 ret = ftrace_modify_code(ip, old, new, true);
243         }
244 #endif
245
246         return ret;
247 }
248
249 int __init ftrace_dyn_arch_init(void)
250 {
251         return 0;
252 }
253 #endif /* CONFIG_DYNAMIC_FTRACE */
254
255 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
256 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
257                            unsigned long frame_pointer)
258 {
259         unsigned long return_hooker = (unsigned long) &return_to_handler;
260         unsigned long old;
261
262         if (unlikely(atomic_read(&current->tracing_graph_pause)))
263                 return;
264
265         old = *parent;
266         *parent = return_hooker;
267
268         if (function_graph_enter(old, self_addr, frame_pointer, NULL))
269                 *parent = old;
270 }
271
272 #ifdef CONFIG_DYNAMIC_FTRACE
273 extern unsigned long ftrace_graph_call;
274 extern unsigned long ftrace_graph_call_old;
275 extern void ftrace_graph_caller_old(void);
276 extern unsigned long ftrace_graph_regs_call;
277 extern void ftrace_graph_regs_caller(void);
278
279 static int __ftrace_modify_caller(unsigned long *callsite,
280                                   void (*func) (void), bool enable)
281 {
282         unsigned long caller_fn = (unsigned long) func;
283         unsigned long pc = (unsigned long) callsite;
284         unsigned long branch = arm_gen_branch(pc, caller_fn);
285         unsigned long nop = 0xe1a00000; /* mov r0, r0 */
286         unsigned long old = enable ? nop : branch;
287         unsigned long new = enable ? branch : nop;
288
289         return ftrace_modify_code(pc, old, new, true);
290 }
291
292 static int ftrace_modify_graph_caller(bool enable)
293 {
294         int ret;
295
296         ret = __ftrace_modify_caller(&ftrace_graph_call,
297                                      ftrace_graph_caller,
298                                      enable);
299
300 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
301         if (!ret)
302                 ret = __ftrace_modify_caller(&ftrace_graph_regs_call,
303                                      ftrace_graph_regs_caller,
304                                      enable);
305 #endif
306
307
308 #ifdef CONFIG_OLD_MCOUNT
309         if (!ret)
310                 ret = __ftrace_modify_caller(&ftrace_graph_call_old,
311                                              ftrace_graph_caller_old,
312                                              enable);
313 #endif
314
315         return ret;
316 }
317
318 int ftrace_enable_ftrace_graph_caller(void)
319 {
320         return ftrace_modify_graph_caller(true);
321 }
322
323 int ftrace_disable_ftrace_graph_caller(void)
324 {
325         return ftrace_modify_graph_caller(false);
326 }
327 #endif /* CONFIG_DYNAMIC_FTRACE */
328 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */