arch/xtensa/kernel/smp.c

   1 /*
   2  * Xtensa SMP support functions.
   3  *
   4  * This file is subject to the terms and conditions of the GNU General Public
   5  * License.  See the file "COPYING" in the main directory of this archive
   6  * for more details.
   7  *
   8  * Copyright (C) 2008 - 2013 Tensilica Inc.
   9  *
  10  * Chris Zankel <chris@zankel.net>
  11  * Joe Taylor <joe@tensilica.com>
  12  * Pete Delaney <piet@tensilica.com
  13  */
  14
  15 #include <linux/cpu.h>
  16 #include <linux/cpumask.h>
  17 #include <linux/delay.h>
  18 #include <linux/init.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/irqdomain.h>
  21 #include <linux/irq.h>
  22 #include <linux/kdebug.h>
  23 #include <linux/module.h>
  24 #include <linux/profile.h>
  25 #include <linux/sched/mm.h>
  26 #include <linux/sched/hotplug.h>
  27 #include <linux/sched/task_stack.h>
  28 #include <linux/reboot.h>
  29 #include <linux/seq_file.h>
  30 #include <linux/smp.h>
  31 #include <linux/thread_info.h>
  32
  33 #include <asm/cacheflush.h>
  34 #include <asm/coprocessor.h>
  35 #include <asm/kdebug.h>
  36 #include <asm/mmu_context.h>
  37 #include <asm/mxregs.h>
  38 #include <asm/platform.h>
  39 #include <asm/tlbflush.h>
  40 #include <asm/traps.h>
  41
  42 #ifdef CONFIG_SMP
  43 # if XCHAL_HAVE_S32C1I == 0
  44 #  error "The S32C1I option is required for SMP."
  45 # endif
  46 #endif
  47
  48 static void system_invalidate_dcache_range(unsigned long start,
  49                 unsigned long size);
  50 static void system_flush_invalidate_dcache_range(unsigned long start,
  51                 unsigned long size);
  52
  53 /* IPI (Inter Process Interrupt) */
  54
  55 #define IPI_IRQ 0
  56
  57 static irqreturn_t ipi_interrupt(int irq, void *dev_id);
  58
  59 void ipi_init(void)
  60 {
  61         unsigned irq = irq_create_mapping(NULL, IPI_IRQ);
  62         if (request_irq(irq, ipi_interrupt, IRQF_PERCPU, "ipi", NULL))
  63                 pr_err("Failed to request irq %u (ipi)\n", irq);
  64 }
  65
  66 static inline unsigned int get_core_count(void)
  67 {
  68         /* Bits 18..21 of SYSCFGID contain the core count minus 1. */
  69         unsigned int syscfgid = get_er(SYSCFGID);
  70         return ((syscfgid >> 18) & 0xf) + 1;
  71 }
  72
  73 static inline int get_core_id(void)
  74 {
  75         /* Bits 0...18 of SYSCFGID contain the core id  */
  76         unsigned int core_id = get_er(SYSCFGID);
  77         return core_id & 0x3fff;
  78 }
  79
  80 void __init smp_prepare_cpus(unsigned int max_cpus)
  81 {
  82         unsigned i;
  83
  84         for_each_possible_cpu(i)
  85                 set_cpu_present(i, true);
  86 }
  87
  88 void __init smp_init_cpus(void)
  89 {
  90         unsigned i;
  91         unsigned int ncpus = get_core_count();
  92         unsigned int core_id = get_core_id();
  93
  94         pr_info("%s: Core Count = %d\n", __func__, ncpus);
  95         pr_info("%s: Core Id = %d\n", __func__, core_id);
  96
  97         if (ncpus > NR_CPUS) {
  98                 ncpus = NR_CPUS;
  99                 pr_info("%s: limiting core count by %d\n", __func__, ncpus);
 100         }
 101
 102         for (i = 0; i < ncpus; ++i)
 103                 set_cpu_possible(i, true);
 104 }
 105
 106 void __init smp_prepare_boot_cpu(void)
 107 {
 108         unsigned int cpu = smp_processor_id();
 109         BUG_ON(cpu != 0);
 110         cpu_asid_cache(cpu) = ASID_USER_FIRST;
 111 }
 112
 113 void __init smp_cpus_done(unsigned int max_cpus)
 114 {
 115 }
 116
 117 static int boot_secondary_processors = 1; /* Set with xt-gdb via .xt-gdb */
 118 static DECLARE_COMPLETION(cpu_running);
 119
 120 void secondary_start_kernel(void)
 121 {
 122         struct mm_struct *mm = &init_mm;
 123         unsigned int cpu = smp_processor_id();
 124
 125         init_mmu();
 126
 127 #ifdef CONFIG_DEBUG_MISC
 128         if (boot_secondary_processors == 0) {
 129                 pr_debug("%s: boot_secondary_processors:%d; Hanging cpu:%d\n",
 130                         __func__, boot_secondary_processors, cpu);
 131                 for (;;)
 132                         __asm__ __volatile__ ("waiti " __stringify(LOCKLEVEL));
 133         }
 134
 135         pr_debug("%s: boot_secondary_processors:%d; Booting cpu:%d\n",
 136                 __func__, boot_secondary_processors, cpu);
 137 #endif
 138         /* Init EXCSAVE1 */
 139
 140         secondary_trap_init();
 141
 142         /* All kernel threads share the same mm context. */
 143
 144         mmget(mm);
 145         mmgrab(mm);
 146         current->active_mm = mm;
 147         cpumask_set_cpu(cpu, mm_cpumask(mm));
 148         enter_lazy_tlb(mm, current);
 149
 150         trace_hardirqs_off();
 151
 152         calibrate_delay();
 153
 154         notify_cpu_starting(cpu);
 155
 156         secondary_init_irq();
 157         local_timer_setup(cpu);
 158
 159         set_cpu_online(cpu, true);
 160
 161         local_irq_enable();
 162
 163         complete(&cpu_running);
 164
 165         cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
 166 }
 167
 168 static void mx_cpu_start(void *p)
 169 {
 170         unsigned cpu = (unsigned)p;
 171         unsigned long run_stall_mask = get_er(MPSCORE);
 172
 173         set_er(run_stall_mask & ~(1u << cpu), MPSCORE);
 174         pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
 175                         __func__, cpu, run_stall_mask, get_er(MPSCORE));
 176 }
 177
 178 static void mx_cpu_stop(void *p)
 179 {
 180         unsigned cpu = (unsigned)p;
 181         unsigned long run_stall_mask = get_er(MPSCORE);
 182
 183         set_er(run_stall_mask | (1u << cpu), MPSCORE);
 184         pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
 185                         __func__, cpu, run_stall_mask, get_er(MPSCORE));
 186 }
 187
 188 #ifdef CONFIG_HOTPLUG_CPU
 189 unsigned long cpu_start_id __cacheline_aligned;
 190 #endif
 191 unsigned long cpu_start_ccount;
 192
 193 static int boot_secondary(unsigned int cpu, struct task_struct *ts)
 194 {
 195         unsigned long timeout = jiffies + msecs_to_jiffies(1000);
 196         unsigned long ccount;
 197         int i;
 198
 199 #ifdef CONFIG_HOTPLUG_CPU
 200         WRITE_ONCE(cpu_start_id, cpu);
 201         /* Pairs with the third memw in the cpu_restart */
 202         mb();
 203         system_flush_invalidate_dcache_range((unsigned long)&cpu_start_id,
 204                                              sizeof(cpu_start_id));
 205 #endif
 206         smp_call_function_single(0, mx_cpu_start, (void *)cpu, 1);
 207
 208         for (i = 0; i < 2; ++i) {
 209                 do
 210                         ccount = get_ccount();
 211                 while (!ccount);
 212
 213                 WRITE_ONCE(cpu_start_ccount, ccount);
 214
 215                 do {
 216                         /*
 217                          * Pairs with the first two memws in the
 218                          * .Lboot_secondary.
 219                          */
 220                         mb();
 221                         ccount = READ_ONCE(cpu_start_ccount);
 222                 } while (ccount && time_before(jiffies, timeout));
 223
 224                 if (ccount) {
 225                         smp_call_function_single(0, mx_cpu_stop,
 226                                                  (void *)cpu, 1);
 227                         WRITE_ONCE(cpu_start_ccount, 0);
 228                         return -EIO;
 229                 }
 230         }
 231         return 0;
 232 }
 233
 234 int __cpu_up(unsigned int cpu, struct task_struct *idle)
 235 {
 236         int ret = 0;
 237
 238         if (cpu_asid_cache(cpu) == 0)
 239                 cpu_asid_cache(cpu) = ASID_USER_FIRST;
 240
 241         start_info.stack = (unsigned long)task_pt_regs(idle);
 242         wmb();
 243
 244         pr_debug("%s: Calling wakeup_secondary(cpu:%d, idle:%p, sp: %08lx)\n",
 245                         __func__, cpu, idle, start_info.stack);
 246
 247         init_completion(&cpu_running);
 248         ret = boot_secondary(cpu, idle);
 249         if (ret == 0) {
 250                 wait_for_completion_timeout(&cpu_running,
 251                                 msecs_to_jiffies(1000));
 252                 if (!cpu_online(cpu))
 253                         ret = -EIO;
 254         }
 255
 256         if (ret)
 257                 pr_err("CPU %u failed to boot\n", cpu);
 258
 259         return ret;
 260 }
 261
 262 #ifdef CONFIG_HOTPLUG_CPU
 263
 264 /*
 265  * __cpu_disable runs on the processor to be shutdown.
 266  */
 267 int __cpu_disable(void)
 268 {
 269         unsigned int cpu = smp_processor_id();
 270
 271         /*
 272          * Take this CPU offline.  Once we clear this, we can't return,
 273          * and we must not schedule until we're ready to give up the cpu.
 274          */
 275         set_cpu_online(cpu, false);
 276
 277 #if XTENSA_HAVE_COPROCESSORS
 278         /*
 279          * Flush coprocessor contexts that are active on the current CPU.
 280          */
 281         local_coprocessors_flush_release_all();
 282 #endif
 283         /*
 284          * OK - migrate IRQs away from this CPU
 285          */
 286         migrate_irqs();
 287
 288         /*
 289          * Flush user cache and TLB mappings, and then remove this CPU
 290          * from the vm mask set of all processes.
 291          */
 292         local_flush_cache_all();
 293         local_flush_tlb_all();
 294         invalidate_page_directory();
 295
 296         clear_tasks_mm_cpumask(cpu);
 297
 298         return 0;
 299 }
 300
 301 static void platform_cpu_kill(unsigned int cpu)
 302 {
 303         smp_call_function_single(0, mx_cpu_stop, (void *)cpu, true);
 304 }
 305
 306 /*
 307  * called on the thread which is asking for a CPU to be shutdown -
 308  * waits until shutdown has completed, or it is timed out.
 309  */
 310 void __cpu_die(unsigned int cpu)
 311 {
 312         unsigned long timeout = jiffies + msecs_to_jiffies(1000);
 313         while (time_before(jiffies, timeout)) {
 314                 system_invalidate_dcache_range((unsigned long)&cpu_start_id,
 315                                                sizeof(cpu_start_id));
 316                 /* Pairs with the second memw in the cpu_restart */
 317                 mb();
 318                 if (READ_ONCE(cpu_start_id) == -cpu) {
 319                         platform_cpu_kill(cpu);
 320                         return;
 321                 }
 322         }
 323         pr_err("CPU%u: unable to kill\n", cpu);
 324 }
 325
 326 void __noreturn arch_cpu_idle_dead(void)
 327 {
 328         cpu_die();
 329 }
 330 /*
 331  * Called from the idle thread for the CPU which has been shutdown.
 332  *
 333  * Note that we disable IRQs here, but do not re-enable them
 334  * before returning to the caller. This is also the behaviour
 335  * of the other hotplug-cpu capable cores, so presumably coming
 336  * out of idle fixes this.
 337  */
 338 void __ref cpu_die(void)
 339 {
 340         idle_task_exit();
 341         local_irq_disable();
 342         __asm__ __volatile__(
 343                         "       movi    a2, cpu_restart\n"
 344                         "       jx      a2\n");
 345
 346         BUG();
 347 }
 348
 349 #endif /* CONFIG_HOTPLUG_CPU */
 350
 351 enum ipi_msg_type {
 352         IPI_RESCHEDULE = 0,
 353         IPI_CALL_FUNC,
 354         IPI_CPU_STOP,
 355         IPI_MAX
 356 };
 357
 358 static const struct {
 359         const char *short_text;
 360         const char *long_text;
 361 } ipi_text[] = {
 362         { .short_text = "RES", .long_text = "Rescheduling interrupts" },
 363         { .short_text = "CAL", .long_text = "Function call interrupts" },
 364         { .short_text = "DIE", .long_text = "CPU shutdown interrupts" },
 365 };
 366
 367 struct ipi_data {
 368         unsigned long ipi_count[IPI_MAX];
 369 };
 370
 371 static DEFINE_PER_CPU(struct ipi_data, ipi_data);
 372
 373 static void send_ipi_message(const struct cpumask *callmask,
 374                 enum ipi_msg_type msg_id)
 375 {
 376         int index;
 377         unsigned long mask = 0;
 378
 379         for_each_cpu(index, callmask)
 380                 mask |= 1 << index;
 381
 382         set_er(mask, MIPISET(msg_id));
 383 }
 384
 385 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
 386 {
 387         send_ipi_message(mask, IPI_CALL_FUNC);
 388 }
 389
 390 void arch_send_call_function_single_ipi(int cpu)
 391 {
 392         send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC);
 393 }
 394
 395 void arch_smp_send_reschedule(int cpu)
 396 {
 397         send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
 398 }
 399
 400 void smp_send_stop(void)
 401 {
 402         struct cpumask targets;
 403
 404         cpumask_copy(&targets, cpu_online_mask);
 405         cpumask_clear_cpu(smp_processor_id(), &targets);
 406         send_ipi_message(&targets, IPI_CPU_STOP);
 407 }
 408
 409 static void ipi_cpu_stop(unsigned int cpu)
 410 {
 411         set_cpu_online(cpu, false);
 412         machine_halt();
 413 }
 414
 415 irqreturn_t ipi_interrupt(int irq, void *dev_id)
 416 {
 417         unsigned int cpu = smp_processor_id();
 418         struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
 419
 420         for (;;) {
 421                 unsigned int msg;
 422
 423                 msg = get_er(MIPICAUSE(cpu));
 424                 set_er(msg, MIPICAUSE(cpu));
 425
 426                 if (!msg)
 427                         break;
 428
 429                 if (msg & (1 << IPI_CALL_FUNC)) {
 430                         ++ipi->ipi_count[IPI_CALL_FUNC];
 431                         generic_smp_call_function_interrupt();
 432                 }
 433
 434                 if (msg & (1 << IPI_RESCHEDULE)) {
 435                         ++ipi->ipi_count[IPI_RESCHEDULE];
 436                         scheduler_ipi();
 437                 }
 438
 439                 if (msg & (1 << IPI_CPU_STOP)) {
 440                         ++ipi->ipi_count[IPI_CPU_STOP];
 441                         ipi_cpu_stop(cpu);
 442                 }
 443         }
 444
 445         return IRQ_HANDLED;
 446 }
 447
 448 void show_ipi_list(struct seq_file *p, int prec)
 449 {
 450         unsigned int cpu;
 451         unsigned i;
 452
 453         for (i = 0; i < IPI_MAX; ++i) {
 454                 seq_printf(p, "%*s:", prec, ipi_text[i].short_text);
 455                 for_each_online_cpu(cpu)
 456                         seq_printf(p, " %10lu",
 457                                         per_cpu(ipi_data, cpu).ipi_count[i]);
 458                 seq_printf(p, "   %s\n", ipi_text[i].long_text);
 459         }
 460 }
 461
 462 int setup_profiling_timer(unsigned int multiplier)
 463 {
 464         pr_debug("setup_profiling_timer %d\n", multiplier);
 465         return 0;
 466 }
 467
 468 /* TLB flush functions */
 469
 470 struct flush_data {
 471         struct vm_area_struct *vma;
 472         unsigned long addr1;
 473         unsigned long addr2;
 474 };
 475
 476 static void ipi_flush_tlb_all(void *arg)
 477 {
 478         local_flush_tlb_all();
 479 }
 480
 481 void flush_tlb_all(void)
 482 {
 483         on_each_cpu(ipi_flush_tlb_all, NULL, 1);
 484 }
 485
 486 static void ipi_flush_tlb_mm(void *arg)
 487 {
 488         local_flush_tlb_mm(arg);
 489 }
 490
 491 void flush_tlb_mm(struct mm_struct *mm)
 492 {
 493         on_each_cpu(ipi_flush_tlb_mm, mm, 1);
 494 }
 495
 496 static void ipi_flush_tlb_page(void *arg)
 497 {
 498         struct flush_data *fd = arg;
 499         local_flush_tlb_page(fd->vma, fd->addr1);
 500 }
 501
 502 void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
 503 {
 504         struct flush_data fd = {
 505                 .vma = vma,
 506                 .addr1 = addr,
 507         };
 508         on_each_cpu(ipi_flush_tlb_page, &fd, 1);
 509 }
 510
 511 static void ipi_flush_tlb_range(void *arg)
 512 {
 513         struct flush_data *fd = arg;
 514         local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2);
 515 }
 516
 517 void flush_tlb_range(struct vm_area_struct *vma,
 518                      unsigned long start, unsigned long end)
 519 {
 520         struct flush_data fd = {
 521                 .vma = vma,
 522                 .addr1 = start,
 523                 .addr2 = end,
 524         };
 525         on_each_cpu(ipi_flush_tlb_range, &fd, 1);
 526 }
 527
 528 static void ipi_flush_tlb_kernel_range(void *arg)
 529 {
 530         struct flush_data *fd = arg;
 531         local_flush_tlb_kernel_range(fd->addr1, fd->addr2);
 532 }
 533
 534 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 535 {
 536         struct flush_data fd = {
 537                 .addr1 = start,
 538                 .addr2 = end,
 539         };
 540         on_each_cpu(ipi_flush_tlb_kernel_range, &fd, 1);
 541 }
 542
 543 /* Cache flush functions */
 544
 545 static void ipi_flush_cache_all(void *arg)
 546 {
 547         local_flush_cache_all();
 548 }
 549
 550 void flush_cache_all(void)
 551 {
 552         on_each_cpu(ipi_flush_cache_all, NULL, 1);
 553 }
 554
 555 static void ipi_flush_cache_page(void *arg)
 556 {
 557         struct flush_data *fd = arg;
 558         local_flush_cache_page(fd->vma, fd->addr1, fd->addr2);
 559 }
 560
 561 void flush_cache_page(struct vm_area_struct *vma,
 562                      unsigned long address, unsigned long pfn)
 563 {
 564         struct flush_data fd = {
 565                 .vma = vma,
 566                 .addr1 = address,
 567                 .addr2 = pfn,
 568         };
 569         on_each_cpu(ipi_flush_cache_page, &fd, 1);
 570 }
 571
 572 static void ipi_flush_cache_range(void *arg)
 573 {
 574         struct flush_data *fd = arg;
 575         local_flush_cache_range(fd->vma, fd->addr1, fd->addr2);
 576 }
 577
 578 void flush_cache_range(struct vm_area_struct *vma,
 579                      unsigned long start, unsigned long end)
 580 {
 581         struct flush_data fd = {
 582                 .vma = vma,
 583                 .addr1 = start,
 584                 .addr2 = end,
 585         };
 586         on_each_cpu(ipi_flush_cache_range, &fd, 1);
 587 }
 588
 589 static void ipi_flush_icache_range(void *arg)
 590 {
 591         struct flush_data *fd = arg;
 592         local_flush_icache_range(fd->addr1, fd->addr2);
 593 }
 594
 595 void flush_icache_range(unsigned long start, unsigned long end)
 596 {
 597         struct flush_data fd = {
 598                 .addr1 = start,
 599                 .addr2 = end,
 600         };
 601         on_each_cpu(ipi_flush_icache_range, &fd, 1);
 602 }
 603 EXPORT_SYMBOL(flush_icache_range);
 604
 605 /* ------------------------------------------------------------------------- */
 606
 607 static void ipi_invalidate_dcache_range(void *arg)
 608 {
 609         struct flush_data *fd = arg;
 610         __invalidate_dcache_range(fd->addr1, fd->addr2);
 611 }
 612
 613 static void system_invalidate_dcache_range(unsigned long start,
 614                 unsigned long size)
 615 {
 616         struct flush_data fd = {
 617                 .addr1 = start,
 618                 .addr2 = size,
 619         };
 620         on_each_cpu(ipi_invalidate_dcache_range, &fd, 1);
 621 }
 622
 623 static void ipi_flush_invalidate_dcache_range(void *arg)
 624 {
 625         struct flush_data *fd = arg;
 626         __flush_invalidate_dcache_range(fd->addr1, fd->addr2);
 627 }
 628
 629 static void system_flush_invalidate_dcache_range(unsigned long start,
 630                 unsigned long size)
 631 {
 632         struct flush_data fd = {
 633                 .addr1 = start,
 634                 .addr2 = size,
 635         };
 636         on_each_cpu(ipi_flush_invalidate_dcache_range, &fd, 1);
 637 }