arch/arm/kernel/ftrace.c

   1 /*
   2  * Dynamic function tracing support.
   3  *
   4  * Copyright (C) 2008 Abhishek Sagar <sagar.abhishek@gmail.com>
   5  * Copyright (C) 2010 Rabin Vincent <rabin@rab.in>
   6  *
   7  * For licencing details, see COPYING.
   8  *
   9  * Defines low-level handling of mcount calls when the kernel
  10  * is compiled with the -pg flag. When using dynamic ftrace, the
  11  * mcount call-sites get patched with NOP till they are enabled.
  12  * All code mutation routines here are called under stop_machine().
  13  */
  14
  15 #include <linux/ftrace.h>
  16 #include <linux/uaccess.h>
  17 #include <linux/module.h>
  18 #include <linux/stop_machine.h>
  19
  20 #include <asm/cacheflush.h>
  21 #include <asm/opcodes.h>
  22 #include <asm/ftrace.h>
  23 #include <asm/insn.h>
  24 #include <asm/set_memory.h>
  25
  26 #ifdef CONFIG_THUMB2_KERNEL
  27 #define NOP             0xf85deb04      /* pop.w {lr} */
  28 #else
  29 #define NOP             0xe8bd4000      /* pop {lr} */
  30 #endif
  31
  32 #ifdef CONFIG_DYNAMIC_FTRACE
  33
  34 static int __ftrace_modify_code(void *data)
  35 {
  36         int *command = data;
  37
  38         set_kernel_text_rw();
  39         ftrace_modify_all_code(*command);
  40         set_kernel_text_ro();
  41
  42         return 0;
  43 }
  44
  45 void arch_ftrace_update_code(int command)
  46 {
  47         stop_machine(__ftrace_modify_code, &command, NULL);
  48 }
  49
  50 static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec)
  51 {
  52         return NOP;
  53 }
  54
  55 static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr)
  56 {
  57         return addr;
  58 }
  59
  60 int ftrace_arch_code_modify_prepare(void)
  61 {
  62         set_all_modules_text_rw();
  63         return 0;
  64 }
  65
  66 int ftrace_arch_code_modify_post_process(void)
  67 {
  68         set_all_modules_text_ro();
  69         /* Make sure any TLB misses during machine stop are cleared. */
  70         flush_tlb_all();
  71         return 0;
  72 }
  73
  74 static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr,
  75                                          bool warn)
  76 {
  77         return arm_gen_branch_link(pc, addr, warn);
  78 }
  79
  80 static int ftrace_modify_code(unsigned long pc, unsigned long old,
  81                               unsigned long new, bool validate)
  82 {
  83         unsigned long replaced;
  84
  85         if (IS_ENABLED(CONFIG_THUMB2_KERNEL)) {
  86                 old = __opcode_to_mem_thumb32(old);
  87                 new = __opcode_to_mem_thumb32(new);
  88         } else {
  89                 old = __opcode_to_mem_arm(old);
  90                 new = __opcode_to_mem_arm(new);
  91         }
  92
  93         if (validate) {
  94                 if (probe_kernel_read(&replaced, (void *)pc, MCOUNT_INSN_SIZE))
  95                         return -EFAULT;
  96
  97                 if (replaced != old)
  98                         return -EINVAL;
  99         }
 100
 101         if (probe_kernel_write((void *)pc, &new, MCOUNT_INSN_SIZE))
 102                 return -EPERM;
 103
 104         flush_icache_range(pc, pc + MCOUNT_INSN_SIZE);
 105
 106         return 0;
 107 }
 108
 109 int ftrace_update_ftrace_func(ftrace_func_t func)
 110 {
 111         unsigned long pc;
 112         unsigned long new;
 113         int ret;
 114
 115         pc = (unsigned long)&ftrace_call;
 116         new = ftrace_call_replace(pc, (unsigned long)func, true);
 117
 118         ret = ftrace_modify_code(pc, 0, new, false);
 119
 120 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 121         if (!ret) {
 122                 pc = (unsigned long)&ftrace_regs_call;
 123                 new = ftrace_call_replace(pc, (unsigned long)func, true);
 124
 125                 ret = ftrace_modify_code(pc, 0, new, false);
 126         }
 127 #endif
 128
 129         return ret;
 130 }
 131
 132 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
 133 {
 134         unsigned long new, old;
 135         unsigned long ip = rec->ip;
 136         unsigned long aaddr = adjust_address(rec, addr);
 137         struct module *mod = NULL;
 138
 139 #ifdef CONFIG_ARM_MODULE_PLTS
 140         mod = rec->arch.mod;
 141 #endif
 142
 143         old = ftrace_nop_replace(rec);
 144
 145         new = ftrace_call_replace(ip, aaddr, !mod);
 146 #ifdef CONFIG_ARM_MODULE_PLTS
 147         if (!new && mod) {
 148                 aaddr = get_module_plt(mod, ip, aaddr);
 149                 new = ftrace_call_replace(ip, aaddr, true);
 150         }
 151 #endif
 152
 153         return ftrace_modify_code(rec->ip, old, new, true);
 154 }
 155
 156 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 157
 158 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
 159                                 unsigned long addr)
 160 {
 161         unsigned long new, old;
 162         unsigned long ip = rec->ip;
 163
 164         old = ftrace_call_replace(ip, adjust_address(rec, old_addr), true);
 165
 166         new = ftrace_call_replace(ip, adjust_address(rec, addr), true);
 167
 168         return ftrace_modify_code(rec->ip, old, new, true);
 169 }
 170
 171 #endif
 172
 173 int ftrace_make_nop(struct module *mod,
 174                     struct dyn_ftrace *rec, unsigned long addr)
 175 {
 176         unsigned long aaddr = adjust_address(rec, addr);
 177         unsigned long ip = rec->ip;
 178         unsigned long old;
 179         unsigned long new;
 180         int ret;
 181
 182 #ifdef CONFIG_ARM_MODULE_PLTS
 183         /* mod is only supplied during module loading */
 184         if (!mod)
 185                 mod = rec->arch.mod;
 186         else
 187                 rec->arch.mod = mod;
 188 #endif
 189
 190         old = ftrace_call_replace(ip, aaddr,
 191                                   !IS_ENABLED(CONFIG_ARM_MODULE_PLTS) || !mod);
 192 #ifdef CONFIG_ARM_MODULE_PLTS
 193         if (!old && mod) {
 194                 aaddr = get_module_plt(mod, ip, aaddr);
 195                 old = ftrace_call_replace(ip, aaddr, true);
 196         }
 197 #endif
 198
 199         new = ftrace_nop_replace(rec);
 200         ret = ftrace_modify_code(ip, old, new, true);
 201
 202         return ret;
 203 }
 204
 205 int __init ftrace_dyn_arch_init(void)
 206 {
 207         return 0;
 208 }
 209 #endif /* CONFIG_DYNAMIC_FTRACE */
 210
 211 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 212 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
 213                            unsigned long frame_pointer)
 214 {
 215         unsigned long return_hooker = (unsigned long) &return_to_handler;
 216         unsigned long old;
 217
 218         if (unlikely(atomic_read(&current->tracing_graph_pause)))
 219                 return;
 220
 221         old = *parent;
 222         *parent = return_hooker;
 223
 224         if (function_graph_enter(old, self_addr, frame_pointer, NULL))
 225                 *parent = old;
 226 }
 227
 228 #ifdef CONFIG_DYNAMIC_FTRACE
 229 extern unsigned long ftrace_graph_call;
 230 extern unsigned long ftrace_graph_call_old;
 231 extern void ftrace_graph_caller_old(void);
 232 extern unsigned long ftrace_graph_regs_call;
 233 extern void ftrace_graph_regs_caller(void);
 234
 235 static int __ftrace_modify_caller(unsigned long *callsite,
 236                                   void (*func) (void), bool enable)
 237 {
 238         unsigned long caller_fn = (unsigned long) func;
 239         unsigned long pc = (unsigned long) callsite;
 240         unsigned long branch = arm_gen_branch(pc, caller_fn);
 241         unsigned long nop = 0xe1a00000; /* mov r0, r0 */
 242         unsigned long old = enable ? nop : branch;
 243         unsigned long new = enable ? branch : nop;
 244
 245         return ftrace_modify_code(pc, old, new, true);
 246 }
 247
 248 static int ftrace_modify_graph_caller(bool enable)
 249 {
 250         int ret;
 251
 252         ret = __ftrace_modify_caller(&ftrace_graph_call,
 253                                      ftrace_graph_caller,
 254                                      enable);
 255
 256 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 257         if (!ret)
 258                 ret = __ftrace_modify_caller(&ftrace_graph_regs_call,
 259                                      ftrace_graph_regs_caller,
 260                                      enable);
 261 #endif
 262
 263
 264         return ret;
 265 }
 266
 267 int ftrace_enable_ftrace_graph_caller(void)
 268 {
 269         return ftrace_modify_graph_caller(true);
 270 }
 271
 272 int ftrace_disable_ftrace_graph_caller(void)
 273 {
 274         return ftrace_modify_graph_caller(false);
 275 }
 276 #endif /* CONFIG_DYNAMIC_FTRACE */
 277 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */