1 // SPDX-License-Identifier: GPL-2.0-only
3 #define pr_fmt(fmt) "callthunks: " fmt
5 #include <linux/debugfs.h>
6 #include <linux/kallsyms.h>
7 #include <linux/memory.h>
8 #include <linux/moduleloader.h>
9 #include <linux/static_call.h>
11 #include <asm/alternative.h>
12 #include <asm/asm-offsets.h>
14 #include <asm/ftrace.h>
16 #include <asm/kexec.h>
17 #include <asm/nospec-branch.h>
18 #include <asm/paravirt.h>
19 #include <asm/sections.h>
20 #include <asm/switch_to.h>
21 #include <asm/sync_core.h>
22 #include <asm/text-patching.h>
23 #include <asm/xen/hypercall.h>
25 static int __initdata_or_module debug_callthunks;
27 #define prdbg(fmt, args...) \
29 if (debug_callthunks) \
30 printk(KERN_DEBUG pr_fmt(fmt), ##args); \
33 static int __init debug_thunks(char *str)
38 __setup("debug-callthunks", debug_thunks);
40 #ifdef CONFIG_CALL_THUNKS_DEBUG
41 DEFINE_PER_CPU(u64, __x86_call_count);
42 DEFINE_PER_CPU(u64, __x86_ret_count);
43 DEFINE_PER_CPU(u64, __x86_stuffs_count);
44 DEFINE_PER_CPU(u64, __x86_ctxsw_count);
45 EXPORT_SYMBOL_GPL(__x86_ctxsw_count);
46 EXPORT_SYMBOL_GPL(__x86_call_count);
49 extern s32 __call_sites[], __call_sites_end[];
57 static bool thunks_initialized __ro_after_init;
59 static const struct core_text builtin_coretext = {
60 .base = (unsigned long)_text,
61 .end = (unsigned long)_etext,
66 ".pushsection .rodata \n"
67 ".global skl_call_thunk_template \n"
68 "skl_call_thunk_template: \n"
69 __stringify(INCREMENT_CALL_DEPTH)" \n"
70 ".global skl_call_thunk_tail \n"
71 "skl_call_thunk_tail: \n"
75 extern u8 skl_call_thunk_template[];
76 extern u8 skl_call_thunk_tail[];
78 #define SKL_TMPL_SIZE \
79 ((unsigned int)(skl_call_thunk_tail - skl_call_thunk_template))
81 extern void error_entry(void);
82 extern void xen_error_entry(void);
83 extern void paranoid_entry(void);
85 static inline bool within_coretext(const struct core_text *ct, void *addr)
87 unsigned long p = (unsigned long)addr;
89 return ct->base <= p && p < ct->end;
92 static inline bool within_module_coretext(void *addr)
100 mod = __module_address((unsigned long)addr);
101 if (mod && within_module_core((unsigned long)addr, mod))
108 static bool is_coretext(const struct core_text *ct, void *addr)
110 if (ct && within_coretext(ct, addr))
112 if (within_coretext(&builtin_coretext, addr))
114 return within_module_coretext(addr);
117 static bool skip_addr(void *dest)
119 if (dest == error_entry)
121 if (dest == paranoid_entry)
123 if (dest == xen_error_entry)
125 /* Does FILL_RSB... */
126 if (dest == __switch_to_asm)
128 /* Accounts directly */
129 if (dest == ret_from_fork)
131 #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_AMD_MEM_ENCRYPT)
132 if (dest == soft_restart_cpu)
135 #ifdef CONFIG_FUNCTION_TRACER
136 if (dest == __fentry__)
139 #ifdef CONFIG_KEXEC_CORE
140 if (dest >= (void *)relocate_kernel &&
141 dest < (void*)relocate_kernel + KEXEC_CONTROL_CODE_MAX_SIZE)
145 if (dest >= (void *)hypercall_page &&
146 dest < (void*)hypercall_page + PAGE_SIZE)
152 static __init_or_module void *call_get_dest(void *addr)
158 ret = insn_decode_kernel(&insn, addr);
162 /* Patched out call? */
163 if (insn.opcode.bytes[0] != CALL_INSN_OPCODE)
166 dest = addr + insn.length + insn.immediate.value;
172 static const u8 nops[] = {
173 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
174 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
175 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
176 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
179 static void *patch_dest(void *dest, bool direct)
181 unsigned int tsize = SKL_TMPL_SIZE;
182 u8 *pad = dest - tsize;
184 /* Already patched? */
185 if (!bcmp(pad, skl_call_thunk_template, tsize))
188 /* Ensure there are nops */
189 if (bcmp(pad, nops, tsize)) {
190 pr_warn_once("Invalid padding area for %pS\n", dest);
195 memcpy(pad, skl_call_thunk_template, tsize);
197 text_poke_copy_locked(pad, skl_call_thunk_template, tsize, true);
201 static __init_or_module void patch_call(void *addr, const struct core_text *ct)
206 if (!within_coretext(ct, addr))
209 dest = call_get_dest(addr);
210 if (!dest || WARN_ON_ONCE(IS_ERR(dest)))
213 if (!is_coretext(ct, dest))
216 pad = patch_dest(dest, within_coretext(ct, dest));
220 prdbg("Patch call at: %pS %px to %pS %px -> %px \n", addr, addr,
222 __text_gen_insn(bytes, CALL_INSN_OPCODE, addr, pad, CALL_INSN_SIZE);
223 text_poke_early(addr, bytes, CALL_INSN_SIZE);
226 static __init_or_module void
227 patch_call_sites(s32 *start, s32 *end, const struct core_text *ct)
231 for (s = start; s < end; s++)
232 patch_call((void *)s + *s, ct);
235 static __init_or_module void
236 patch_alt_call_sites(struct alt_instr *start, struct alt_instr *end,
237 const struct core_text *ct)
241 for (a = start; a < end; a++)
242 patch_call((void *)&a->instr_offset + a->instr_offset, ct);
245 static __init_or_module void
246 callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct)
248 prdbg("Patching call sites %s\n", ct->name);
249 patch_call_sites(cs->call_start, cs->call_end, ct);
250 patch_alt_call_sites(cs->alt_start, cs->alt_end, ct);
251 prdbg("Patching call sites done%s\n", ct->name);
254 void __init callthunks_patch_builtin_calls(void)
256 struct callthunk_sites cs = {
257 .call_start = __call_sites,
258 .call_end = __call_sites_end,
259 .alt_start = __alt_instructions,
260 .alt_end = __alt_instructions_end
263 if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
266 pr_info("Setting up call depth tracking\n");
267 mutex_lock(&text_mutex);
268 callthunks_setup(&cs, &builtin_coretext);
269 thunks_initialized = true;
270 mutex_unlock(&text_mutex);
273 void *callthunks_translate_call_dest(void *dest)
277 lockdep_assert_held(&text_mutex);
279 if (!thunks_initialized || skip_addr(dest))
282 if (!is_coretext(NULL, dest))
285 target = patch_dest(dest, false);
286 return target ? : dest;
289 #ifdef CONFIG_BPF_JIT
290 static bool is_callthunk(void *addr)
292 unsigned int tmpl_size = SKL_TMPL_SIZE;
293 void *tmpl = skl_call_thunk_template;
296 dest = roundup((unsigned long)addr, CONFIG_FUNCTION_ALIGNMENT);
297 if (!thunks_initialized || skip_addr((void *)dest))
300 return !bcmp((void *)(dest - tmpl_size), tmpl, tmpl_size);
303 int x86_call_depth_emit_accounting(u8 **pprog, void *func)
305 unsigned int tmpl_size = SKL_TMPL_SIZE;
306 void *tmpl = skl_call_thunk_template;
308 if (!thunks_initialized)
311 /* Is function call target a thunk? */
312 if (func && is_callthunk(func))
315 memcpy(*pprog, tmpl, tmpl_size);
321 #ifdef CONFIG_MODULES
322 void noinline callthunks_patch_module_calls(struct callthunk_sites *cs,
325 struct core_text ct = {
326 .base = (unsigned long)mod->mem[MOD_TEXT].base,
327 .end = (unsigned long)mod->mem[MOD_TEXT].base + mod->mem[MOD_TEXT].size,
331 if (!thunks_initialized)
334 mutex_lock(&text_mutex);
335 callthunks_setup(cs, &ct);
336 mutex_unlock(&text_mutex);
338 #endif /* CONFIG_MODULES */
340 #if defined(CONFIG_CALL_THUNKS_DEBUG) && defined(CONFIG_DEBUG_FS)
341 static int callthunks_debug_show(struct seq_file *m, void *p)
343 unsigned long cpu = (unsigned long)m->private;
345 seq_printf(m, "C: %16llu R: %16llu S: %16llu X: %16llu\n,",
346 per_cpu(__x86_call_count, cpu),
347 per_cpu(__x86_ret_count, cpu),
348 per_cpu(__x86_stuffs_count, cpu),
349 per_cpu(__x86_ctxsw_count, cpu));
353 static int callthunks_debug_open(struct inode *inode, struct file *file)
355 return single_open(file, callthunks_debug_show, inode->i_private);
358 static const struct file_operations dfs_ops = {
359 .open = callthunks_debug_open,
362 .release = single_release,
365 static int __init callthunks_debugfs_init(void)
370 dir = debugfs_create_dir("callthunks", NULL);
371 for_each_possible_cpu(cpu) {
372 void *arg = (void *)cpu;
375 sprintf(name, "cpu%lu", cpu);
376 debugfs_create_file(name, 0644, dir, arg, &dfs_ops);
380 __initcall(callthunks_debugfs_init);