1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd.
4 #include <linux/ftrace.h>
5 #include <linux/uaccess.h>
6 #include <linux/stop_machine.h>
7 #include <asm/cacheflush.h>
9 #ifdef CONFIG_DYNAMIC_FTRACE
12 #define NOP32_HI 0xc400
13 #define NOP32_LO 0x4820
14 #define PUSH_LR 0x14d0
15 #define MOVIH_LINK 0xea3a
16 #define ORI_LINK 0xef5a
17 #define JSR_LINK 0xe8fa
18 #define BSR_LINK 0xe000
21 * Gcc-csky with -pg will insert stub in function prologue:
27 * If the (callee - current_pc) is less then 64MB, we'll use bsr:
32 * else we'll use (movih + ori + jsr):
38 * (r26 is our reserved link-reg)
41 static inline void make_jbsr(unsigned long callee, unsigned long pc,
42 uint16_t *call, bool nolr)
46 call[0] = nolr ? NOP : PUSH_LR;
48 offset = (long) callee - (long) pc;
50 if (unlikely(offset < -67108864 || offset > 67108864)) {
52 call[2] = callee >> 16;
54 call[4] = callee & 0xffff;
61 ((uint16_t)((unsigned long) offset >> 16) & 0x3ff);
62 call[2] = (uint16_t)((unsigned long) offset & 0xffff);
63 call[3] = call[5] = NOP32_HI;
64 call[4] = call[6] = NOP32_LO;
68 static uint16_t nops[7] = {NOP, NOP32_HI, NOP32_LO, NOP32_HI, NOP32_LO,
70 static int ftrace_check_current_nop(unsigned long hook)
73 unsigned long hook_pos = hook - 2;
75 if (copy_from_kernel_nofault((void *)olds, (void *)hook_pos,
79 if (memcmp((void *)nops, (void *)olds, sizeof(nops))) {
80 pr_err("%p: nop but get (%04x %04x %04x %04x %04x %04x %04x)\n",
82 olds[0], olds[1], olds[2], olds[3], olds[4], olds[5],
91 static int ftrace_modify_code(unsigned long hook, unsigned long target,
92 bool enable, bool nolr)
96 unsigned long hook_pos = hook - 2;
99 make_jbsr(target, hook, call, nolr);
101 ret = copy_to_kernel_nofault((void *)hook_pos, enable ? call : nops,
106 flush_icache_range(hook_pos, hook_pos + MCOUNT_INSN_SIZE);
111 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
113 int ret = ftrace_check_current_nop(rec->ip);
118 return ftrace_modify_code(rec->ip, addr, true, false);
121 int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec,
124 return ftrace_modify_code(rec->ip, addr, false, false);
127 int ftrace_update_ftrace_func(ftrace_func_t func)
129 int ret = ftrace_modify_code((unsigned long)&ftrace_call,
130 (unsigned long)func, true, true);
132 ret = ftrace_modify_code((unsigned long)&ftrace_regs_call,
133 (unsigned long)func, true, true);
137 int __init ftrace_dyn_arch_init(void)
141 #endif /* CONFIG_DYNAMIC_FTRACE */
143 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
144 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
147 return ftrace_modify_code(rec->ip, addr, true, true);
151 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
152 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
153 unsigned long frame_pointer)
155 unsigned long return_hooker = (unsigned long)&return_to_handler;
158 if (unlikely(atomic_read(¤t->tracing_graph_pause)))
163 if (!function_graph_enter(old, self_addr,
164 *(unsigned long *)frame_pointer, parent)) {
166 * For csky-gcc function has sub-call:
170 * st.w r15, (sp, 0x4)
173 * We only need set *parent for resume
175 * For csky-gcc function has no sub-call:
181 * We need set *parent and *(frame_pointer + 4) for resume,
182 * because lr is resumed twice.
184 *parent = return_hooker;
186 if (*(unsigned long *)frame_pointer == old)
187 *(unsigned long *)frame_pointer = return_hooker;
191 #ifdef CONFIG_DYNAMIC_FTRACE
192 int ftrace_enable_ftrace_graph_caller(void)
194 return ftrace_modify_code((unsigned long)&ftrace_graph_call,
195 (unsigned long)&ftrace_graph_caller, true, true);
198 int ftrace_disable_ftrace_graph_caller(void)
200 return ftrace_modify_code((unsigned long)&ftrace_graph_call,
201 (unsigned long)&ftrace_graph_caller, false, true);
203 #endif /* CONFIG_DYNAMIC_FTRACE */
204 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
206 #ifdef CONFIG_DYNAMIC_FTRACE
207 #ifndef CONFIG_CPU_HAS_ICACHE_INS
208 struct ftrace_modify_param {
213 static int __ftrace_modify_code(void *data)
215 struct ftrace_modify_param *param = data;
217 if (atomic_inc_return(¶m->cpu_count) == 1) {
218 ftrace_modify_all_code(param->command);
219 atomic_inc(¶m->cpu_count);
221 while (atomic_read(¶m->cpu_count) <= num_online_cpus())
223 local_icache_inv_all(NULL);
229 void arch_ftrace_update_code(int command)
231 struct ftrace_modify_param param = { command, ATOMIC_INIT(0) };
233 stop_machine(__ftrace_modify_code, ¶m, cpu_online_mask);
236 #endif /* CONFIG_DYNAMIC_FTRACE */
238 /* _mcount is defined in abi's mcount.S */
239 EXPORT_SYMBOL(_mcount);