arch/loongarch/kernel/kprobes.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 #include <linux/kdebug.h>
   3 #include <linux/kprobes.h>
   4 #include <linux/preempt.h>
   5 #include <asm/break.h>
   6
   7 #define KPROBE_BP_INSN          larch_insn_gen_break(BRK_KPROBE_BP)
   8 #define KPROBE_SSTEPBP_INSN     larch_insn_gen_break(BRK_KPROBE_SSTEPBP)
   9
  10 DEFINE_PER_CPU(struct kprobe *, current_kprobe);
  11 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
  12
  13 static void arch_prepare_ss_slot(struct kprobe *p)
  14 {
  15         p->ainsn.insn[0] = *p->addr;
  16         p->ainsn.insn[1] = KPROBE_SSTEPBP_INSN;
  17         p->ainsn.restore = (unsigned long)p->addr + LOONGARCH_INSN_SIZE;
  18 }
  19 NOKPROBE_SYMBOL(arch_prepare_ss_slot);
  20
  21 static void arch_prepare_simulate(struct kprobe *p)
  22 {
  23         p->ainsn.restore = 0;
  24 }
  25 NOKPROBE_SYMBOL(arch_prepare_simulate);
  26
  27 int arch_prepare_kprobe(struct kprobe *p)
  28 {
  29         union loongarch_instruction insn;
  30
  31         if ((unsigned long)p->addr & 0x3)
  32                 return -EILSEQ;
  33
  34         /* copy instruction */
  35         p->opcode = *p->addr;
  36         insn.word = p->opcode;
  37
  38         /* decode instruction */
  39         if (insns_not_supported(insn))
  40                 return -EINVAL;
  41
  42         if (insns_need_simulation(insn)) {
  43                 p->ainsn.insn = NULL;
  44         } else {
  45                 p->ainsn.insn = get_insn_slot();
  46                 if (!p->ainsn.insn)
  47                         return -ENOMEM;
  48         }
  49
  50         /* prepare the instruction */
  51         if (p->ainsn.insn)
  52                 arch_prepare_ss_slot(p);
  53         else
  54                 arch_prepare_simulate(p);
  55
  56         return 0;
  57 }
  58 NOKPROBE_SYMBOL(arch_prepare_kprobe);
  59
  60 /* Install breakpoint in text */
  61 void arch_arm_kprobe(struct kprobe *p)
  62 {
  63         *p->addr = KPROBE_BP_INSN;
  64         flush_insn_slot(p);
  65 }
  66 NOKPROBE_SYMBOL(arch_arm_kprobe);
  67
  68 /* Remove breakpoint from text */
  69 void arch_disarm_kprobe(struct kprobe *p)
  70 {
  71         *p->addr = p->opcode;
  72         flush_insn_slot(p);
  73 }
  74 NOKPROBE_SYMBOL(arch_disarm_kprobe);
  75
  76 void arch_remove_kprobe(struct kprobe *p)
  77 {
  78         if (p->ainsn.insn) {
  79                 free_insn_slot(p->ainsn.insn, 0);
  80                 p->ainsn.insn = NULL;
  81         }
  82 }
  83 NOKPROBE_SYMBOL(arch_remove_kprobe);
  84
  85 static void save_previous_kprobe(struct kprobe_ctlblk *kcb)
  86 {
  87         kcb->prev_kprobe.kp = kprobe_running();
  88         kcb->prev_kprobe.status = kcb->kprobe_status;
  89 }
  90 NOKPROBE_SYMBOL(save_previous_kprobe);
  91
  92 static void restore_previous_kprobe(struct kprobe_ctlblk *kcb)
  93 {
  94         __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
  95         kcb->kprobe_status = kcb->prev_kprobe.status;
  96 }
  97 NOKPROBE_SYMBOL(restore_previous_kprobe);
  98
  99 static void set_current_kprobe(struct kprobe *p)
 100 {
 101         __this_cpu_write(current_kprobe, p);
 102 }
 103 NOKPROBE_SYMBOL(set_current_kprobe);
 104
 105 /*
 106  * Interrupts need to be disabled before single-step mode is set,
 107  * and not reenabled until after single-step mode ends.
 108  * Without disabling interrupt on local CPU, there is a chance of
 109  * interrupt occurrence in the period of exception return and start
 110  * of out-of-line single-step, that result in wrongly single stepping
 111  * into the interrupt handler.
 112  */
 113 static void save_local_irqflag(struct kprobe_ctlblk *kcb,
 114                                struct pt_regs *regs)
 115 {
 116         kcb->saved_status = regs->csr_prmd;
 117         regs->csr_prmd &= ~CSR_PRMD_PIE;
 118 }
 119 NOKPROBE_SYMBOL(save_local_irqflag);
 120
 121 static void restore_local_irqflag(struct kprobe_ctlblk *kcb,
 122                                   struct pt_regs *regs)
 123 {
 124         regs->csr_prmd = kcb->saved_status;
 125 }
 126 NOKPROBE_SYMBOL(restore_local_irqflag);
 127
 128 static void post_kprobe_handler(struct kprobe *cur, struct kprobe_ctlblk *kcb,
 129                                 struct pt_regs *regs)
 130 {
 131         /* return addr restore if non-branching insn */
 132         if (cur->ainsn.restore != 0)
 133                 instruction_pointer_set(regs, cur->ainsn.restore);
 134
 135         /* restore back original saved kprobe variables and continue */
 136         if (kcb->kprobe_status == KPROBE_REENTER) {
 137                 restore_previous_kprobe(kcb);
 138                 preempt_enable_no_resched();
 139                 return;
 140         }
 141
 142         /*
 143          * update the kcb status even if the cur->post_handler is
 144          * not set because reset_curent_kprobe() doesn't update kcb.
 145          */
 146         kcb->kprobe_status = KPROBE_HIT_SSDONE;
 147         if (cur->post_handler)
 148                 cur->post_handler(cur, regs, 0);
 149
 150         reset_current_kprobe();
 151         preempt_enable_no_resched();
 152 }
 153 NOKPROBE_SYMBOL(post_kprobe_handler);
 154
 155 static void setup_singlestep(struct kprobe *p, struct pt_regs *regs,
 156                              struct kprobe_ctlblk *kcb, int reenter)
 157 {
 158         union loongarch_instruction insn;
 159
 160         if (reenter) {
 161                 save_previous_kprobe(kcb);
 162                 set_current_kprobe(p);
 163                 kcb->kprobe_status = KPROBE_REENTER;
 164         } else {
 165                 kcb->kprobe_status = KPROBE_HIT_SS;
 166         }
 167
 168         if (p->ainsn.insn) {
 169                 /* IRQs and single stepping do not mix well */
 170                 save_local_irqflag(kcb, regs);
 171                 /* set ip register to prepare for single stepping */
 172                 regs->csr_era = (unsigned long)p->ainsn.insn;
 173         } else {
 174                 /* simulate single steping */
 175                 insn.word = p->opcode;
 176                 arch_simulate_insn(insn, regs);
 177                 /* now go for post processing */
 178                 post_kprobe_handler(p, kcb, regs);
 179         }
 180 }
 181 NOKPROBE_SYMBOL(setup_singlestep);
 182
 183 static bool reenter_kprobe(struct kprobe *p, struct pt_regs *regs,
 184                            struct kprobe_ctlblk *kcb)
 185 {
 186         switch (kcb->kprobe_status) {
 187         case KPROBE_HIT_SS:
 188         case KPROBE_HIT_SSDONE:
 189         case KPROBE_HIT_ACTIVE:
 190                 kprobes_inc_nmissed_count(p);
 191                 setup_singlestep(p, regs, kcb, 1);
 192                 break;
 193         case KPROBE_REENTER:
 194                 pr_warn("Failed to recover from reentered kprobes.\n");
 195                 dump_kprobe(p);
 196                 WARN_ON_ONCE(1);
 197                 break;
 198         default:
 199                 WARN_ON(1);
 200                 return false;
 201         }
 202
 203         return true;
 204 }
 205 NOKPROBE_SYMBOL(reenter_kprobe);
 206
 207 bool kprobe_breakpoint_handler(struct pt_regs *regs)
 208 {
 209         struct kprobe_ctlblk *kcb;
 210         struct kprobe *p, *cur_kprobe;
 211         kprobe_opcode_t *addr = (kprobe_opcode_t *)regs->csr_era;
 212
 213         /*
 214          * We don't want to be preempted for the entire
 215          * duration of kprobe processing.
 216          */
 217         preempt_disable();
 218         kcb = get_kprobe_ctlblk();
 219         cur_kprobe = kprobe_running();
 220
 221         p = get_kprobe(addr);
 222         if (p) {
 223                 if (cur_kprobe) {
 224                         if (reenter_kprobe(p, regs, kcb))
 225                                 return true;
 226                 } else {
 227                         /* Probe hit */
 228                         set_current_kprobe(p);
 229                         kcb->kprobe_status = KPROBE_HIT_ACTIVE;
 230
 231                         /*
 232                          * If we have no pre-handler or it returned 0, we
 233                          * continue with normal processing.  If we have a
 234                          * pre-handler and it returned non-zero, it will
 235                          * modify the execution path and no need to single
 236                          * stepping. Let's just reset current kprobe and exit.
 237                          *
 238                          * pre_handler can hit a breakpoint and can step thru
 239                          * before return.
 240                          */
 241                         if (!p->pre_handler || !p->pre_handler(p, regs)) {
 242                                 setup_singlestep(p, regs, kcb, 0);
 243                         } else {
 244                                 reset_current_kprobe();
 245                                 preempt_enable_no_resched();
 246                         }
 247                         return true;
 248                 }
 249         }
 250
 251         if (*addr != KPROBE_BP_INSN) {
 252                 /*
 253                  * The breakpoint instruction was removed right
 254                  * after we hit it.  Another cpu has removed
 255                  * either a probepoint or a debugger breakpoint
 256                  * at this address.  In either case, no further
 257                  * handling of this interrupt is appropriate.
 258                  * Return back to original instruction, and continue.
 259                  */
 260                 regs->csr_era = (unsigned long)addr;
 261                 preempt_enable_no_resched();
 262                 return true;
 263         }
 264
 265         preempt_enable_no_resched();
 266         return false;
 267 }
 268 NOKPROBE_SYMBOL(kprobe_breakpoint_handler);
 269
 270 bool kprobe_singlestep_handler(struct pt_regs *regs)
 271 {
 272         struct kprobe *cur = kprobe_running();
 273         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 274         unsigned long addr = instruction_pointer(regs);
 275
 276         if (cur && (kcb->kprobe_status & (KPROBE_HIT_SS | KPROBE_REENTER)) &&
 277             ((unsigned long)&cur->ainsn.insn[1] == addr)) {
 278                 restore_local_irqflag(kcb, regs);
 279                 post_kprobe_handler(cur, kcb, regs);
 280                 return true;
 281         }
 282
 283         preempt_enable_no_resched();
 284         return false;
 285 }
 286 NOKPROBE_SYMBOL(kprobe_singlestep_handler);
 287
 288 bool kprobe_fault_handler(struct pt_regs *regs, int trapnr)
 289 {
 290         struct kprobe *cur = kprobe_running();
 291         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 292
 293         switch (kcb->kprobe_status) {
 294         case KPROBE_HIT_SS:
 295         case KPROBE_REENTER:
 296                 /*
 297                  * We are here because the instruction being single
 298                  * stepped caused a page fault. We reset the current
 299                  * kprobe and the ip points back to the probe address
 300                  * and allow the page fault handler to continue as a
 301                  * normal page fault.
 302                  */
 303                 regs->csr_era = (unsigned long)cur->addr;
 304                 WARN_ON_ONCE(!instruction_pointer(regs));
 305
 306                 if (kcb->kprobe_status == KPROBE_REENTER) {
 307                         restore_previous_kprobe(kcb);
 308                 } else {
 309                         restore_local_irqflag(kcb, regs);
 310                         reset_current_kprobe();
 311                 }
 312                 preempt_enable_no_resched();
 313                 break;
 314         }
 315         return false;
 316 }
 317 NOKPROBE_SYMBOL(kprobe_fault_handler);
 318
 319 /*
 320  * Provide a blacklist of symbols identifying ranges which cannot be kprobed.
 321  * This blacklist is exposed to userspace via debugfs (kprobes/blacklist).
 322  */
 323 int __init arch_populate_kprobe_blacklist(void)
 324 {
 325         return kprobe_add_area_blacklist((unsigned long)__irqentry_text_start,
 326                                          (unsigned long)__irqentry_text_end);
 327 }
 328
 329 int __init arch_init_kprobes(void)
 330 {
 331         return 0;
 332 }
 333
 334 int arch_trampoline_kprobe(struct kprobe *p)
 335 {
 336         return 0;
 337 }
 338 NOKPROBE_SYMBOL(arch_trampoline_kprobe);