arch/arm64/kernel/stacktrace.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Stack tracing support
   4  *
   5  * Copyright (C) 2012 ARM Ltd.
   6  */
   7 #include <linux/kernel.h>
   8 #include <linux/efi.h>
   9 #include <linux/export.h>
  10 #include <linux/filter.h>
  11 #include <linux/ftrace.h>
  12 #include <linux/kprobes.h>
  13 #include <linux/sched.h>
  14 #include <linux/sched/debug.h>
  15 #include <linux/sched/task_stack.h>
  16 #include <linux/stacktrace.h>
  17
  18 #include <asm/efi.h>
  19 #include <asm/irq.h>
  20 #include <asm/stack_pointer.h>
  21 #include <asm/stacktrace.h>
  22
  23 /*
  24  * Kernel unwind state
  25  *
  26  * @common:      Common unwind state.
  27  * @task:        The task being unwound.
  28  * @kr_cur:      When KRETPROBES is selected, holds the kretprobe instance
  29  *               associated with the most recently encountered replacement lr
  30  *               value.
  31  */
  32 struct kunwind_state {
  33         struct unwind_state common;
  34         struct task_struct *task;
  35 #ifdef CONFIG_KRETPROBES
  36         struct llist_node *kr_cur;
  37 #endif
  38 };
  39
  40 static __always_inline void
  41 kunwind_init(struct kunwind_state *state,
  42              struct task_struct *task)
  43 {
  44         unwind_init_common(&state->common);
  45         state->task = task;
  46 }
  47
  48 /*
  49  * Start an unwind from a pt_regs.
  50  *
  51  * The unwind will begin at the PC within the regs.
  52  *
  53  * The regs must be on a stack currently owned by the calling task.
  54  */
  55 static __always_inline void
  56 kunwind_init_from_regs(struct kunwind_state *state,
  57                        struct pt_regs *regs)
  58 {
  59         kunwind_init(state, current);
  60
  61         state->common.fp = regs->regs[29];
  62         state->common.pc = regs->pc;
  63 }
  64
  65 /*
  66  * Start an unwind from a caller.
  67  *
  68  * The unwind will begin at the caller of whichever function this is inlined
  69  * into.
  70  *
  71  * The function which invokes this must be noinline.
  72  */
  73 static __always_inline void
  74 kunwind_init_from_caller(struct kunwind_state *state)
  75 {
  76         kunwind_init(state, current);
  77
  78         state->common.fp = (unsigned long)__builtin_frame_address(1);
  79         state->common.pc = (unsigned long)__builtin_return_address(0);
  80 }
  81
  82 /*
  83  * Start an unwind from a blocked task.
  84  *
  85  * The unwind will begin at the blocked tasks saved PC (i.e. the caller of
  86  * cpu_switch_to()).
  87  *
  88  * The caller should ensure the task is blocked in cpu_switch_to() for the
  89  * duration of the unwind, or the unwind will be bogus. It is never valid to
  90  * call this for the current task.
  91  */
  92 static __always_inline void
  93 kunwind_init_from_task(struct kunwind_state *state,
  94                        struct task_struct *task)
  95 {
  96         kunwind_init(state, task);
  97
  98         state->common.fp = thread_saved_fp(task);
  99         state->common.pc = thread_saved_pc(task);
 100 }
 101
 102 static __always_inline int
 103 kunwind_recover_return_address(struct kunwind_state *state)
 104 {
 105 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 106         if (state->task->ret_stack &&
 107             (state->common.pc == (unsigned long)return_to_handler)) {
 108                 unsigned long orig_pc;
 109                 orig_pc = ftrace_graph_ret_addr(state->task, NULL,
 110                                                 state->common.pc,
 111                                                 (void *)state->common.fp);
 112                 if (WARN_ON_ONCE(state->common.pc == orig_pc))
 113                         return -EINVAL;
 114                 state->common.pc = orig_pc;
 115         }
 116 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 117
 118 #ifdef CONFIG_KRETPROBES
 119         if (is_kretprobe_trampoline(state->common.pc)) {
 120                 unsigned long orig_pc;
 121                 orig_pc = kretprobe_find_ret_addr(state->task,
 122                                                   (void *)state->common.fp,
 123                                                   &state->kr_cur);
 124                 state->common.pc = orig_pc;
 125         }
 126 #endif /* CONFIG_KRETPROBES */
 127
 128         return 0;
 129 }
 130
 131 /*
 132  * Unwind from one frame record (A) to the next frame record (B).
 133  *
 134  * We terminate early if the location of B indicates a malformed chain of frame
 135  * records (e.g. a cycle), determined based on the location and fp value of A
 136  * and the location (but not the fp value) of B.
 137  */
 138 static __always_inline int
 139 kunwind_next(struct kunwind_state *state)
 140 {
 141         struct task_struct *tsk = state->task;
 142         unsigned long fp = state->common.fp;
 143         int err;
 144
 145         /* Final frame; nothing to unwind */
 146         if (fp == (unsigned long)task_pt_regs(tsk)->stackframe)
 147                 return -ENOENT;
 148
 149         err = unwind_next_frame_record(&state->common);
 150         if (err)
 151                 return err;
 152
 153         state->common.pc = ptrauth_strip_kernel_insn_pac(state->common.pc);
 154
 155         return kunwind_recover_return_address(state);
 156 }
 157
 158 typedef bool (*kunwind_consume_fn)(const struct kunwind_state *state, void *cookie);
 159
 160 static __always_inline void
 161 do_kunwind(struct kunwind_state *state, kunwind_consume_fn consume_state,
 162            void *cookie)
 163 {
 164         if (kunwind_recover_return_address(state))
 165                 return;
 166
 167         while (1) {
 168                 int ret;
 169
 170                 if (!consume_state(state, cookie))
 171                         break;
 172                 ret = kunwind_next(state);
 173                 if (ret < 0)
 174                         break;
 175         }
 176 }
 177
 178 /*
 179  * Per-cpu stacks are only accessible when unwinding the current task in a
 180  * non-preemptible context.
 181  */
 182 #define STACKINFO_CPU(name)                                     \
 183         ({                                                      \
 184                 ((task == current) && !preemptible())           \
 185                         ? stackinfo_get_##name()                \
 186                         : stackinfo_get_unknown();              \
 187         })
 188
 189 /*
 190  * SDEI stacks are only accessible when unwinding the current task in an NMI
 191  * context.
 192  */
 193 #define STACKINFO_SDEI(name)                                    \
 194         ({                                                      \
 195                 ((task == current) && in_nmi())                 \
 196                         ? stackinfo_get_sdei_##name()           \
 197                         : stackinfo_get_unknown();              \
 198         })
 199
 200 #define STACKINFO_EFI                                           \
 201         ({                                                      \
 202                 ((task == current) && current_in_efi())         \
 203                         ? stackinfo_get_efi()                   \
 204                         : stackinfo_get_unknown();              \
 205         })
 206
 207 static __always_inline void
 208 kunwind_stack_walk(kunwind_consume_fn consume_state,
 209                    void *cookie, struct task_struct *task,
 210                    struct pt_regs *regs)
 211 {
 212         struct stack_info stacks[] = {
 213                 stackinfo_get_task(task),
 214                 STACKINFO_CPU(irq),
 215 #if defined(CONFIG_VMAP_STACK)
 216                 STACKINFO_CPU(overflow),
 217 #endif
 218 #if defined(CONFIG_VMAP_STACK) && defined(CONFIG_ARM_SDE_INTERFACE)
 219                 STACKINFO_SDEI(normal),
 220                 STACKINFO_SDEI(critical),
 221 #endif
 222 #ifdef CONFIG_EFI
 223                 STACKINFO_EFI,
 224 #endif
 225         };
 226         struct kunwind_state state = {
 227                 .common = {
 228                         .stacks = stacks,
 229                         .nr_stacks = ARRAY_SIZE(stacks),
 230                 },
 231         };
 232
 233         if (regs) {
 234                 if (task != current)
 235                         return;
 236                 kunwind_init_from_regs(&state, regs);
 237         } else if (task == current) {
 238                 kunwind_init_from_caller(&state);
 239         } else {
 240                 kunwind_init_from_task(&state, task);
 241         }
 242
 243         do_kunwind(&state, consume_state, cookie);
 244 }
 245
 246 struct kunwind_consume_entry_data {
 247         stack_trace_consume_fn consume_entry;
 248         void *cookie;
 249 };
 250
 251 static __always_inline bool
 252 arch_kunwind_consume_entry(const struct kunwind_state *state, void *cookie)
 253 {
 254         struct kunwind_consume_entry_data *data = cookie;
 255         return data->consume_entry(data->cookie, state->common.pc);
 256 }
 257
 258 noinline noinstr void arch_stack_walk(stack_trace_consume_fn consume_entry,
 259                               void *cookie, struct task_struct *task,
 260                               struct pt_regs *regs)
 261 {
 262         struct kunwind_consume_entry_data data = {
 263                 .consume_entry = consume_entry,
 264                 .cookie = cookie,
 265         };
 266
 267         kunwind_stack_walk(arch_kunwind_consume_entry, &data, task, regs);
 268 }
 269
 270 struct bpf_unwind_consume_entry_data {
 271         bool (*consume_entry)(void *cookie, u64 ip, u64 sp, u64 fp);
 272         void *cookie;
 273 };
 274
 275 static bool
 276 arch_bpf_unwind_consume_entry(const struct kunwind_state *state, void *cookie)
 277 {
 278         struct bpf_unwind_consume_entry_data *data = cookie;
 279
 280         return data->consume_entry(data->cookie, state->common.pc, 0,
 281                                    state->common.fp);
 282 }
 283
 284 noinline noinstr void arch_bpf_stack_walk(bool (*consume_entry)(void *cookie, u64 ip, u64 sp,
 285                                                                 u64 fp), void *cookie)
 286 {
 287         struct bpf_unwind_consume_entry_data data = {
 288                 .consume_entry = consume_entry,
 289                 .cookie = cookie,
 290         };
 291
 292         kunwind_stack_walk(arch_bpf_unwind_consume_entry, &data, current, NULL);
 293 }
 294
 295 static bool dump_backtrace_entry(void *arg, unsigned long where)
 296 {
 297         char *loglvl = arg;
 298         printk("%s %pSb\n", loglvl, (void *)where);
 299         return true;
 300 }
 301
 302 void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk,
 303                     const char *loglvl)
 304 {
 305         pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk);
 306
 307         if (regs && user_mode(regs))
 308                 return;
 309
 310         if (!tsk)
 311                 tsk = current;
 312
 313         if (!try_get_task_stack(tsk))
 314                 return;
 315
 316         printk("%sCall trace:\n", loglvl);
 317         arch_stack_walk(dump_backtrace_entry, (void *)loglvl, tsk, regs);
 318
 319         put_task_stack(tsk);
 320 }
 321
 322 void show_stack(struct task_struct *tsk, unsigned long *sp, const char *loglvl)
 323 {
 324         dump_backtrace(NULL, tsk, loglvl);
 325         barrier();
 326 }