arch/powerpc/kernel/hw_breakpoint.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
   4  * using the CPU's debug registers. Derived from
   5  * "arch/x86/kernel/hw_breakpoint.c"
   6  *
   7  * Copyright 2010 IBM Corporation
   8  * Author: K.Prasad <prasad@linux.vnet.ibm.com>
   9  */
  10
  11 #include <linux/hw_breakpoint.h>
  12 #include <linux/notifier.h>
  13 #include <linux/kprobes.h>
  14 #include <linux/percpu.h>
  15 #include <linux/kernel.h>
  16 #include <linux/sched.h>
  17 #include <linux/smp.h>
  18 #include <linux/spinlock.h>
  19 #include <linux/debugfs.h>
  20 #include <linux/init.h>
  21
  22 #include <asm/hw_breakpoint.h>
  23 #include <asm/processor.h>
  24 #include <asm/sstep.h>
  25 #include <asm/debug.h>
  26 #include <asm/hvcall.h>
  27 #include <asm/inst.h>
  28 #include <linux/uaccess.h>
  29
  30 /*
  31  * Stores the breakpoints currently in use on each breakpoint address
  32  * register for every cpu
  33  */
  34 static DEFINE_PER_CPU(struct perf_event *, bp_per_reg[HBP_NUM_MAX]);
  35
  36 /*
  37  * Returns total number of data or instruction breakpoints available.
  38  */
  39 int hw_breakpoint_slots(int type)
  40 {
  41         if (type == TYPE_DATA)
  42                 return nr_wp_slots();
  43         return 0;               /* no instruction breakpoints available */
  44 }
  45
  46 static bool single_step_pending(void)
  47 {
  48         int i;
  49
  50         for (i = 0; i < nr_wp_slots(); i++) {
  51                 if (current->thread.last_hit_ubp[i])
  52                         return true;
  53         }
  54         return false;
  55 }
  56
  57 /*
  58  * Install a perf counter breakpoint.
  59  *
  60  * We seek a free debug address register and use it for this
  61  * breakpoint.
  62  *
  63  * Atomic: we hold the counter->ctx->lock and we only handle variables
  64  * and registers local to this cpu.
  65  */
  66 int arch_install_hw_breakpoint(struct perf_event *bp)
  67 {
  68         struct arch_hw_breakpoint *info = counter_arch_bp(bp);
  69         struct perf_event **slot;
  70         int i;
  71
  72         for (i = 0; i < nr_wp_slots(); i++) {
  73                 slot = this_cpu_ptr(&bp_per_reg[i]);
  74                 if (!*slot) {
  75                         *slot = bp;
  76                         break;
  77                 }
  78         }
  79
  80         if (WARN_ONCE(i == nr_wp_slots(), "Can't find any breakpoint slot"))
  81                 return -EBUSY;
  82
  83         /*
  84          * Do not install DABR values if the instruction must be single-stepped.
  85          * If so, DABR will be populated in single_step_dabr_instruction().
  86          */
  87         if (!single_step_pending())
  88                 __set_breakpoint(i, info);
  89
  90         return 0;
  91 }
  92
  93 /*
  94  * Uninstall the breakpoint contained in the given counter.
  95  *
  96  * First we search the debug address register it uses and then we disable
  97  * it.
  98  *
  99  * Atomic: we hold the counter->ctx->lock and we only handle variables
 100  * and registers local to this cpu.
 101  */
 102 void arch_uninstall_hw_breakpoint(struct perf_event *bp)
 103 {
 104         struct arch_hw_breakpoint null_brk = {0};
 105         struct perf_event **slot;
 106         int i;
 107
 108         for (i = 0; i < nr_wp_slots(); i++) {
 109                 slot = this_cpu_ptr(&bp_per_reg[i]);
 110                 if (*slot == bp) {
 111                         *slot = NULL;
 112                         break;
 113                 }
 114         }
 115
 116         if (WARN_ONCE(i == nr_wp_slots(), "Can't find any breakpoint slot"))
 117                 return;
 118
 119         __set_breakpoint(i, &null_brk);
 120 }
 121
 122 static bool is_ptrace_bp(struct perf_event *bp)
 123 {
 124         return bp->overflow_handler == ptrace_triggered;
 125 }
 126
 127 struct breakpoint {
 128         struct list_head list;
 129         struct perf_event *bp;
 130         bool ptrace_bp;
 131 };
 132
 133 /*
 134  * While kernel/events/hw_breakpoint.c does its own synchronization, we cannot
 135  * rely on it safely synchronizing internals here; however, we can rely on it
 136  * not requesting more breakpoints than available.
 137  */
 138 static DEFINE_SPINLOCK(cpu_bps_lock);
 139 static DEFINE_PER_CPU(struct breakpoint *, cpu_bps[HBP_NUM_MAX]);
 140 static DEFINE_SPINLOCK(task_bps_lock);
 141 static LIST_HEAD(task_bps);
 142
 143 static struct breakpoint *alloc_breakpoint(struct perf_event *bp)
 144 {
 145         struct breakpoint *tmp;
 146
 147         tmp = kzalloc(sizeof(*tmp), GFP_KERNEL);
 148         if (!tmp)
 149                 return ERR_PTR(-ENOMEM);
 150         tmp->bp = bp;
 151         tmp->ptrace_bp = is_ptrace_bp(bp);
 152         return tmp;
 153 }
 154
 155 static bool bp_addr_range_overlap(struct perf_event *bp1, struct perf_event *bp2)
 156 {
 157         __u64 bp1_saddr, bp1_eaddr, bp2_saddr, bp2_eaddr;
 158
 159         bp1_saddr = ALIGN_DOWN(bp1->attr.bp_addr, HW_BREAKPOINT_SIZE);
 160         bp1_eaddr = ALIGN(bp1->attr.bp_addr + bp1->attr.bp_len, HW_BREAKPOINT_SIZE);
 161         bp2_saddr = ALIGN_DOWN(bp2->attr.bp_addr, HW_BREAKPOINT_SIZE);
 162         bp2_eaddr = ALIGN(bp2->attr.bp_addr + bp2->attr.bp_len, HW_BREAKPOINT_SIZE);
 163
 164         return (bp1_saddr < bp2_eaddr && bp1_eaddr > bp2_saddr);
 165 }
 166
 167 static bool alternate_infra_bp(struct breakpoint *b, struct perf_event *bp)
 168 {
 169         return is_ptrace_bp(bp) ? !b->ptrace_bp : b->ptrace_bp;
 170 }
 171
 172 static bool can_co_exist(struct breakpoint *b, struct perf_event *bp)
 173 {
 174         return !(alternate_infra_bp(b, bp) && bp_addr_range_overlap(b->bp, bp));
 175 }
 176
 177 static int task_bps_add(struct perf_event *bp)
 178 {
 179         struct breakpoint *tmp;
 180
 181         tmp = alloc_breakpoint(bp);
 182         if (IS_ERR(tmp))
 183                 return PTR_ERR(tmp);
 184
 185         spin_lock(&task_bps_lock);
 186         list_add(&tmp->list, &task_bps);
 187         spin_unlock(&task_bps_lock);
 188         return 0;
 189 }
 190
 191 static void task_bps_remove(struct perf_event *bp)
 192 {
 193         struct list_head *pos, *q;
 194
 195         spin_lock(&task_bps_lock);
 196         list_for_each_safe(pos, q, &task_bps) {
 197                 struct breakpoint *tmp = list_entry(pos, struct breakpoint, list);
 198
 199                 if (tmp->bp == bp) {
 200                         list_del(&tmp->list);
 201                         kfree(tmp);
 202                         break;
 203                 }
 204         }
 205         spin_unlock(&task_bps_lock);
 206 }
 207
 208 /*
 209  * If any task has breakpoint from alternate infrastructure,
 210  * return true. Otherwise return false.
 211  */
 212 static bool all_task_bps_check(struct perf_event *bp)
 213 {
 214         struct breakpoint *tmp;
 215         bool ret = false;
 216
 217         spin_lock(&task_bps_lock);
 218         list_for_each_entry(tmp, &task_bps, list) {
 219                 if (!can_co_exist(tmp, bp)) {
 220                         ret = true;
 221                         break;
 222                 }
 223         }
 224         spin_unlock(&task_bps_lock);
 225         return ret;
 226 }
 227
 228 /*
 229  * If same task has breakpoint from alternate infrastructure,
 230  * return true. Otherwise return false.
 231  */
 232 static bool same_task_bps_check(struct perf_event *bp)
 233 {
 234         struct breakpoint *tmp;
 235         bool ret = false;
 236
 237         spin_lock(&task_bps_lock);
 238         list_for_each_entry(tmp, &task_bps, list) {
 239                 if (tmp->bp->hw.target == bp->hw.target &&
 240                     !can_co_exist(tmp, bp)) {
 241                         ret = true;
 242                         break;
 243                 }
 244         }
 245         spin_unlock(&task_bps_lock);
 246         return ret;
 247 }
 248
 249 static int cpu_bps_add(struct perf_event *bp)
 250 {
 251         struct breakpoint **cpu_bp;
 252         struct breakpoint *tmp;
 253         int i = 0;
 254
 255         tmp = alloc_breakpoint(bp);
 256         if (IS_ERR(tmp))
 257                 return PTR_ERR(tmp);
 258
 259         spin_lock(&cpu_bps_lock);
 260         cpu_bp = per_cpu_ptr(cpu_bps, bp->cpu);
 261         for (i = 0; i < nr_wp_slots(); i++) {
 262                 if (!cpu_bp[i]) {
 263                         cpu_bp[i] = tmp;
 264                         break;
 265                 }
 266         }
 267         spin_unlock(&cpu_bps_lock);
 268         return 0;
 269 }
 270
 271 static void cpu_bps_remove(struct perf_event *bp)
 272 {
 273         struct breakpoint **cpu_bp;
 274         int i = 0;
 275
 276         spin_lock(&cpu_bps_lock);
 277         cpu_bp = per_cpu_ptr(cpu_bps, bp->cpu);
 278         for (i = 0; i < nr_wp_slots(); i++) {
 279                 if (!cpu_bp[i])
 280                         continue;
 281
 282                 if (cpu_bp[i]->bp == bp) {
 283                         kfree(cpu_bp[i]);
 284                         cpu_bp[i] = NULL;
 285                         break;
 286                 }
 287         }
 288         spin_unlock(&cpu_bps_lock);
 289 }
 290
 291 static bool cpu_bps_check(int cpu, struct perf_event *bp)
 292 {
 293         struct breakpoint **cpu_bp;
 294         bool ret = false;
 295         int i;
 296
 297         spin_lock(&cpu_bps_lock);
 298         cpu_bp = per_cpu_ptr(cpu_bps, cpu);
 299         for (i = 0; i < nr_wp_slots(); i++) {
 300                 if (cpu_bp[i] && !can_co_exist(cpu_bp[i], bp)) {
 301                         ret = true;
 302                         break;
 303                 }
 304         }
 305         spin_unlock(&cpu_bps_lock);
 306         return ret;
 307 }
 308
 309 static bool all_cpu_bps_check(struct perf_event *bp)
 310 {
 311         int cpu;
 312
 313         for_each_online_cpu(cpu) {
 314                 if (cpu_bps_check(cpu, bp))
 315                         return true;
 316         }
 317         return false;
 318 }
 319
 320 int arch_reserve_bp_slot(struct perf_event *bp)
 321 {
 322         int ret;
 323
 324         /* ptrace breakpoint */
 325         if (is_ptrace_bp(bp)) {
 326                 if (all_cpu_bps_check(bp))
 327                         return -ENOSPC;
 328
 329                 if (same_task_bps_check(bp))
 330                         return -ENOSPC;
 331
 332                 return task_bps_add(bp);
 333         }
 334
 335         /* perf breakpoint */
 336         if (is_kernel_addr(bp->attr.bp_addr))
 337                 return 0;
 338
 339         if (bp->hw.target && bp->cpu == -1) {
 340                 if (same_task_bps_check(bp))
 341                         return -ENOSPC;
 342
 343                 return task_bps_add(bp);
 344         } else if (!bp->hw.target && bp->cpu != -1) {
 345                 if (all_task_bps_check(bp))
 346                         return -ENOSPC;
 347
 348                 return cpu_bps_add(bp);
 349         }
 350
 351         if (same_task_bps_check(bp))
 352                 return -ENOSPC;
 353
 354         ret = cpu_bps_add(bp);
 355         if (ret)
 356                 return ret;
 357         ret = task_bps_add(bp);
 358         if (ret)
 359                 cpu_bps_remove(bp);
 360
 361         return ret;
 362 }
 363
 364 void arch_release_bp_slot(struct perf_event *bp)
 365 {
 366         if (!is_kernel_addr(bp->attr.bp_addr)) {
 367                 if (bp->hw.target)
 368                         task_bps_remove(bp);
 369                 if (bp->cpu != -1)
 370                         cpu_bps_remove(bp);
 371         }
 372 }
 373
 374 /*
 375  * Perform cleanup of arch-specific counters during unregistration
 376  * of the perf-event
 377  */
 378 void arch_unregister_hw_breakpoint(struct perf_event *bp)
 379 {
 380         /*
 381          * If the breakpoint is unregistered between a hw_breakpoint_handler()
 382          * and the single_step_dabr_instruction(), then cleanup the breakpoint
 383          * restoration variables to prevent dangling pointers.
 384          * FIXME, this should not be using bp->ctx at all! Sayeth peterz.
 385          */
 386         if (bp->ctx && bp->ctx->task && bp->ctx->task != ((void *)-1L)) {
 387                 int i;
 388
 389                 for (i = 0; i < nr_wp_slots(); i++) {
 390                         if (bp->ctx->task->thread.last_hit_ubp[i] == bp)
 391                                 bp->ctx->task->thread.last_hit_ubp[i] = NULL;
 392                 }
 393         }
 394 }
 395
 396 /*
 397  * Check for virtual address in kernel space.
 398  */
 399 int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw)
 400 {
 401         return is_kernel_addr(hw->address);
 402 }
 403
 404 int arch_bp_generic_fields(int type, int *gen_bp_type)
 405 {
 406         *gen_bp_type = 0;
 407         if (type & HW_BRK_TYPE_READ)
 408                 *gen_bp_type |= HW_BREAKPOINT_R;
 409         if (type & HW_BRK_TYPE_WRITE)
 410                 *gen_bp_type |= HW_BREAKPOINT_W;
 411         if (*gen_bp_type == 0)
 412                 return -EINVAL;
 413         return 0;
 414 }
 415
 416 /*
 417  * Watchpoint match range is always doubleword(8 bytes) aligned on
 418  * powerpc. If the given range is crossing doubleword boundary, we
 419  * need to increase the length such that next doubleword also get
 420  * covered. Ex,
 421  *
 422  *          address   len = 6 bytes
 423  *                |=========.
 424  *   |------------v--|------v--------|
 425  *   | | | | | | | | | | | | | | | | |
 426  *   |---------------|---------------|
 427  *    <---8 bytes--->
 428  *
 429  * In this case, we should configure hw as:
 430  *   start_addr = address & ~(HW_BREAKPOINT_SIZE - 1)
 431  *   len = 16 bytes
 432  *
 433  * @start_addr is inclusive but @end_addr is exclusive.
 434  */
 435 static int hw_breakpoint_validate_len(struct arch_hw_breakpoint *hw)
 436 {
 437         u16 max_len = DABR_MAX_LEN;
 438         u16 hw_len;
 439         unsigned long start_addr, end_addr;
 440
 441         start_addr = ALIGN_DOWN(hw->address, HW_BREAKPOINT_SIZE);
 442         end_addr = ALIGN(hw->address + hw->len, HW_BREAKPOINT_SIZE);
 443         hw_len = end_addr - start_addr;
 444
 445         if (dawr_enabled()) {
 446                 max_len = DAWR_MAX_LEN;
 447                 /* DAWR region can't cross 512 bytes boundary on p10 predecessors */
 448                 if (!cpu_has_feature(CPU_FTR_ARCH_31) &&
 449                     (ALIGN_DOWN(start_addr, SZ_512) != ALIGN_DOWN(end_addr - 1, SZ_512)))
 450                         return -EINVAL;
 451         } else if (IS_ENABLED(CONFIG_PPC_8xx)) {
 452                 /* 8xx can setup a range without limitation */
 453                 max_len = U16_MAX;
 454         }
 455
 456         if (hw_len > max_len)
 457                 return -EINVAL;
 458
 459         hw->hw_len = hw_len;
 460         return 0;
 461 }
 462
 463 /*
 464  * Validate the arch-specific HW Breakpoint register settings
 465  */
 466 int hw_breakpoint_arch_parse(struct perf_event *bp,
 467                              const struct perf_event_attr *attr,
 468                              struct arch_hw_breakpoint *hw)
 469 {
 470         int ret = -EINVAL;
 471
 472         if (!bp || !attr->bp_len)
 473                 return ret;
 474
 475         hw->type = HW_BRK_TYPE_TRANSLATE;
 476         if (attr->bp_type & HW_BREAKPOINT_R)
 477                 hw->type |= HW_BRK_TYPE_READ;
 478         if (attr->bp_type & HW_BREAKPOINT_W)
 479                 hw->type |= HW_BRK_TYPE_WRITE;
 480         if (hw->type == HW_BRK_TYPE_TRANSLATE)
 481                 /* must set alteast read or write */
 482                 return ret;
 483         if (!attr->exclude_user)
 484                 hw->type |= HW_BRK_TYPE_USER;
 485         if (!attr->exclude_kernel)
 486                 hw->type |= HW_BRK_TYPE_KERNEL;
 487         if (!attr->exclude_hv)
 488                 hw->type |= HW_BRK_TYPE_HYP;
 489         hw->address = attr->bp_addr;
 490         hw->len = attr->bp_len;
 491
 492         if (!ppc_breakpoint_available())
 493                 return -ENODEV;
 494
 495         return hw_breakpoint_validate_len(hw);
 496 }
 497
 498 /*
 499  * Restores the breakpoint on the debug registers.
 500  * Invoke this function if it is known that the execution context is
 501  * about to change to cause loss of MSR_SE settings.
 502  */
 503 void thread_change_pc(struct task_struct *tsk, struct pt_regs *regs)
 504 {
 505         struct arch_hw_breakpoint *info;
 506         int i;
 507
 508         for (i = 0; i < nr_wp_slots(); i++) {
 509                 if (unlikely(tsk->thread.last_hit_ubp[i]))
 510                         goto reset;
 511         }
 512         return;
 513
 514 reset:
 515         regs_set_return_msr(regs, regs->msr & ~MSR_SE);
 516         for (i = 0; i < nr_wp_slots(); i++) {
 517                 info = counter_arch_bp(__this_cpu_read(bp_per_reg[i]));
 518                 __set_breakpoint(i, info);
 519                 tsk->thread.last_hit_ubp[i] = NULL;
 520         }
 521 }
 522
 523 static bool is_larx_stcx_instr(int type)
 524 {
 525         return type == LARX || type == STCX;
 526 }
 527
 528 static bool is_octword_vsx_instr(int type, int size)
 529 {
 530         return ((type == LOAD_VSX || type == STORE_VSX) && size == 32);
 531 }
 532
 533 /*
 534  * We've failed in reliably handling the hw-breakpoint. Unregister
 535  * it and throw a warning message to let the user know about it.
 536  */
 537 static void handler_error(struct perf_event *bp, struct arch_hw_breakpoint *info)
 538 {
 539         WARN(1, "Unable to handle hardware breakpoint. Breakpoint at 0x%lx will be disabled.",
 540              info->address);
 541         perf_event_disable_inatomic(bp);
 542 }
 543
 544 static void larx_stcx_err(struct perf_event *bp, struct arch_hw_breakpoint *info)
 545 {
 546         printk_ratelimited("Breakpoint hit on instruction that can't be emulated. Breakpoint at 0x%lx will be disabled.\n",
 547                            info->address);
 548         perf_event_disable_inatomic(bp);
 549 }
 550
 551 static bool stepping_handler(struct pt_regs *regs, struct perf_event **bp,
 552                              struct arch_hw_breakpoint **info, int *hit,
 553                              ppc_inst_t instr)
 554 {
 555         int i;
 556         int stepped;
 557
 558         /* Do not emulate user-space instructions, instead single-step them */
 559         if (user_mode(regs)) {
 560                 for (i = 0; i < nr_wp_slots(); i++) {
 561                         if (!hit[i])
 562                                 continue;
 563                         current->thread.last_hit_ubp[i] = bp[i];
 564                         info[i] = NULL;
 565                 }
 566                 regs_set_return_msr(regs, regs->msr | MSR_SE);
 567                 return false;
 568         }
 569
 570         stepped = emulate_step(regs, instr);
 571         if (!stepped) {
 572                 for (i = 0; i < nr_wp_slots(); i++) {
 573                         if (!hit[i])
 574                                 continue;
 575                         handler_error(bp[i], info[i]);
 576                         info[i] = NULL;
 577                 }
 578                 return false;
 579         }
 580         return true;
 581 }
 582
 583 static void handle_p10dd1_spurious_exception(struct arch_hw_breakpoint **info,
 584                                              int *hit, unsigned long ea)
 585 {
 586         int i;
 587         unsigned long hw_end_addr;
 588
 589         /*
 590          * Handle spurious exception only when any bp_per_reg is set.
 591          * Otherwise this might be created by xmon and not actually a
 592          * spurious exception.
 593          */
 594         for (i = 0; i < nr_wp_slots(); i++) {
 595                 if (!info[i])
 596                         continue;
 597
 598                 hw_end_addr = ALIGN(info[i]->address + info[i]->len, HW_BREAKPOINT_SIZE);
 599
 600                 /*
 601                  * Ending address of DAWR range is less than starting
 602                  * address of op.
 603                  */
 604                 if ((hw_end_addr - 1) >= ea)
 605                         continue;
 606
 607                 /*
 608                  * Those addresses need to be in the same or in two
 609                  * consecutive 512B blocks;
 610                  */
 611                 if (((hw_end_addr - 1) >> 10) != (ea >> 10))
 612                         continue;
 613
 614                 /*
 615                  * 'op address + 64B' generates an address that has a
 616                  * carry into bit 52 (crosses 2K boundary).
 617                  */
 618                 if ((ea & 0x800) == ((ea + 64) & 0x800))
 619                         continue;
 620
 621                 break;
 622         }
 623
 624         if (i == nr_wp_slots())
 625                 return;
 626
 627         for (i = 0; i < nr_wp_slots(); i++) {
 628                 if (info[i]) {
 629                         hit[i] = 1;
 630                         info[i]->type |= HW_BRK_TYPE_EXTRANEOUS_IRQ;
 631                 }
 632         }
 633 }
 634
 635 int hw_breakpoint_handler(struct die_args *args)
 636 {
 637         bool err = false;
 638         int rc = NOTIFY_STOP;
 639         struct perf_event *bp[HBP_NUM_MAX] = { NULL };
 640         struct pt_regs *regs = args->regs;
 641         struct arch_hw_breakpoint *info[HBP_NUM_MAX] = { NULL };
 642         int i;
 643         int hit[HBP_NUM_MAX] = {0};
 644         int nr_hit = 0;
 645         bool ptrace_bp = false;
 646         ppc_inst_t instr = ppc_inst(0);
 647         int type = 0;
 648         int size = 0;
 649         unsigned long ea;
 650
 651         /* Disable breakpoints during exception handling */
 652         hw_breakpoint_disable();
 653
 654         /*
 655          * The counter may be concurrently released but that can only
 656          * occur from a call_rcu() path. We can then safely fetch
 657          * the breakpoint, use its callback, touch its counter
 658          * while we are in an rcu_read_lock() path.
 659          */
 660         rcu_read_lock();
 661
 662         if (!IS_ENABLED(CONFIG_PPC_8xx))
 663                 wp_get_instr_detail(regs, &instr, &type, &size, &ea);
 664
 665         for (i = 0; i < nr_wp_slots(); i++) {
 666                 bp[i] = __this_cpu_read(bp_per_reg[i]);
 667                 if (!bp[i])
 668                         continue;
 669
 670                 info[i] = counter_arch_bp(bp[i]);
 671                 info[i]->type &= ~HW_BRK_TYPE_EXTRANEOUS_IRQ;
 672
 673                 if (wp_check_constraints(regs, instr, ea, type, size, info[i])) {
 674                         if (!IS_ENABLED(CONFIG_PPC_8xx) &&
 675                             ppc_inst_equal(instr, ppc_inst(0))) {
 676                                 handler_error(bp[i], info[i]);
 677                                 info[i] = NULL;
 678                                 err = 1;
 679                                 continue;
 680                         }
 681
 682                         if (is_ptrace_bp(bp[i]))
 683                                 ptrace_bp = true;
 684                         hit[i] = 1;
 685                         nr_hit++;
 686                 }
 687         }
 688
 689         if (err)
 690                 goto reset;
 691
 692         if (!nr_hit) {
 693                 /* Workaround for Power10 DD1 */
 694                 if (!IS_ENABLED(CONFIG_PPC_8xx) && mfspr(SPRN_PVR) == 0x800100 &&
 695                     is_octword_vsx_instr(type, size)) {
 696                         handle_p10dd1_spurious_exception(info, hit, ea);
 697                 } else {
 698                         rc = NOTIFY_DONE;
 699                         goto out;
 700                 }
 701         }
 702
 703         /*
 704          * Return early after invoking user-callback function without restoring
 705          * DABR if the breakpoint is from ptrace which always operates in
 706          * one-shot mode. The ptrace-ed process will receive the SIGTRAP signal
 707          * generated in do_dabr().
 708          */
 709         if (ptrace_bp) {
 710                 for (i = 0; i < nr_wp_slots(); i++) {
 711                         if (!hit[i])
 712                                 continue;
 713                         perf_bp_event(bp[i], regs);
 714                         info[i] = NULL;
 715                 }
 716                 rc = NOTIFY_DONE;
 717                 goto reset;
 718         }
 719
 720         if (!IS_ENABLED(CONFIG_PPC_8xx)) {
 721                 if (is_larx_stcx_instr(type)) {
 722                         for (i = 0; i < nr_wp_slots(); i++) {
 723                                 if (!hit[i])
 724                                         continue;
 725                                 larx_stcx_err(bp[i], info[i]);
 726                                 info[i] = NULL;
 727                         }
 728                         goto reset;
 729                 }
 730
 731                 if (!stepping_handler(regs, bp, info, hit, instr))
 732                         goto reset;
 733         }
 734
 735         /*
 736          * As a policy, the callback is invoked in a 'trigger-after-execute'
 737          * fashion
 738          */
 739         for (i = 0; i < nr_wp_slots(); i++) {
 740                 if (!hit[i])
 741                         continue;
 742                 if (!(info[i]->type & HW_BRK_TYPE_EXTRANEOUS_IRQ))
 743                         perf_bp_event(bp[i], regs);
 744         }
 745
 746 reset:
 747         for (i = 0; i < nr_wp_slots(); i++) {
 748                 if (!info[i])
 749                         continue;
 750                 __set_breakpoint(i, info[i]);
 751         }
 752
 753 out:
 754         rcu_read_unlock();
 755         return rc;
 756 }
 757 NOKPROBE_SYMBOL(hw_breakpoint_handler);
 758
 759 /*
 760  * Handle single-step exceptions following a DABR hit.
 761  */
 762 static int single_step_dabr_instruction(struct die_args *args)
 763 {
 764         struct pt_regs *regs = args->regs;
 765         struct perf_event *bp = NULL;
 766         struct arch_hw_breakpoint *info;
 767         int i;
 768         bool found = false;
 769
 770         /*
 771          * Check if we are single-stepping as a result of a
 772          * previous HW Breakpoint exception
 773          */
 774         for (i = 0; i < nr_wp_slots(); i++) {
 775                 bp = current->thread.last_hit_ubp[i];
 776
 777                 if (!bp)
 778                         continue;
 779
 780                 found = true;
 781                 info = counter_arch_bp(bp);
 782
 783                 /*
 784                  * We shall invoke the user-defined callback function in the
 785                  * single stepping handler to confirm to 'trigger-after-execute'
 786                  * semantics
 787                  */
 788                 if (!(info->type & HW_BRK_TYPE_EXTRANEOUS_IRQ))
 789                         perf_bp_event(bp, regs);
 790                 current->thread.last_hit_ubp[i] = NULL;
 791         }
 792
 793         if (!found)
 794                 return NOTIFY_DONE;
 795
 796         for (i = 0; i < nr_wp_slots(); i++) {
 797                 bp = __this_cpu_read(bp_per_reg[i]);
 798                 if (!bp)
 799                         continue;
 800
 801                 info = counter_arch_bp(bp);
 802                 __set_breakpoint(i, info);
 803         }
 804
 805         /*
 806          * If the process was being single-stepped by ptrace, let the
 807          * other single-step actions occur (e.g. generate SIGTRAP).
 808          */
 809         if (test_thread_flag(TIF_SINGLESTEP))
 810                 return NOTIFY_DONE;
 811
 812         return NOTIFY_STOP;
 813 }
 814 NOKPROBE_SYMBOL(single_step_dabr_instruction);
 815
 816 /*
 817  * Handle debug exception notifications.
 818  */
 819 int hw_breakpoint_exceptions_notify(
 820                 struct notifier_block *unused, unsigned long val, void *data)
 821 {
 822         int ret = NOTIFY_DONE;
 823
 824         switch (val) {
 825         case DIE_DABR_MATCH:
 826                 ret = hw_breakpoint_handler(data);
 827                 break;
 828         case DIE_SSTEP:
 829                 ret = single_step_dabr_instruction(data);
 830                 break;
 831         }
 832
 833         return ret;
 834 }
 835 NOKPROBE_SYMBOL(hw_breakpoint_exceptions_notify);
 836
 837 /*
 838  * Release the user breakpoints used by ptrace
 839  */
 840 void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
 841 {
 842         int i;
 843         struct thread_struct *t = &tsk->thread;
 844
 845         for (i = 0; i < nr_wp_slots(); i++) {
 846                 unregister_hw_breakpoint(t->ptrace_bps[i]);
 847                 t->ptrace_bps[i] = NULL;
 848         }
 849 }
 850
 851 void hw_breakpoint_pmu_read(struct perf_event *bp)
 852 {
 853         /* TODO */
 854 }
 855
 856 void ptrace_triggered(struct perf_event *bp,
 857                       struct perf_sample_data *data, struct pt_regs *regs)
 858 {
 859         struct perf_event_attr attr;
 860
 861         /*
 862          * Disable the breakpoint request here since ptrace has defined a
 863          * one-shot behaviour for breakpoint exceptions in PPC64.
 864          * The SIGTRAP signal is generated automatically for us in do_dabr().
 865          * We don't have to do anything about that here
 866          */
 867         attr = bp->attr;
 868         attr.disabled = true;
 869         modify_user_hw_breakpoint(bp, &attr);
 870 }