drivers/irqchip/irq-sifive-plic.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2017 SiFive
   4  * Copyright (C) 2018 Christoph Hellwig
   5  */
   6 #define pr_fmt(fmt) "plic: " fmt
   7 #include <linux/cpu.h>
   8 #include <linux/interrupt.h>
   9 #include <linux/io.h>
  10 #include <linux/irq.h>
  11 #include <linux/irqchip.h>
  12 #include <linux/irqchip/chained_irq.h>
  13 #include <linux/irqdomain.h>
  14 #include <linux/module.h>
  15 #include <linux/of.h>
  16 #include <linux/of_address.h>
  17 #include <linux/of_irq.h>
  18 #include <linux/platform_device.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/syscore_ops.h>
  21 #include <asm/smp.h>
  22
  23 /*
  24  * This driver implements a version of the RISC-V PLIC with the actual layout
  25  * specified in chapter 8 of the SiFive U5 Coreplex Series Manual:
  26  *
  27  *     https://static.dev.sifive.com/U54-MC-RVCoreIP.pdf
  28  *
  29  * The largest number supported by devices marked as 'sifive,plic-1.0.0', is
  30  * 1024, of which device 0 is defined as non-existent by the RISC-V Privileged
  31  * Spec.
  32  */
  33
  34 #define MAX_DEVICES                     1024
  35 #define MAX_CONTEXTS                    15872
  36
  37 /*
  38  * Each interrupt source has a priority register associated with it.
  39  * We always hardwire it to one in Linux.
  40  */
  41 #define PRIORITY_BASE                   0
  42 #define     PRIORITY_PER_ID             4
  43
  44 /*
  45  * Each hart context has a vector of interrupt enable bits associated with it.
  46  * There's one bit for each interrupt source.
  47  */
  48 #define CONTEXT_ENABLE_BASE             0x2000
  49 #define     CONTEXT_ENABLE_SIZE         0x80
  50
  51 /*
  52  * Each hart context has a set of control registers associated with it.  Right
  53  * now there's only two: a source priority threshold over which the hart will
  54  * take an interrupt, and a register to claim interrupts.
  55  */
  56 #define CONTEXT_BASE                    0x200000
  57 #define     CONTEXT_SIZE                0x1000
  58 #define     CONTEXT_THRESHOLD           0x00
  59 #define     CONTEXT_CLAIM               0x04
  60
  61 #define PLIC_DISABLE_THRESHOLD          0x7
  62 #define PLIC_ENABLE_THRESHOLD           0
  63
  64 #define PLIC_QUIRK_EDGE_INTERRUPT       0
  65
  66 struct plic_priv {
  67         struct cpumask lmask;
  68         struct irq_domain *irqdomain;
  69         void __iomem *regs;
  70         unsigned long plic_quirks;
  71         unsigned int nr_irqs;
  72         unsigned long *prio_save;
  73 };
  74
  75 struct plic_handler {
  76         bool                    present;
  77         void __iomem            *hart_base;
  78         /*
  79          * Protect mask operations on the registers given that we can't
  80          * assume atomic memory operations work on them.
  81          */
  82         raw_spinlock_t          enable_lock;
  83         void __iomem            *enable_base;
  84         u32                     *enable_save;
  85         struct plic_priv        *priv;
  86 };
  87 static int plic_parent_irq __ro_after_init;
  88 static bool plic_cpuhp_setup_done __ro_after_init;
  89 static DEFINE_PER_CPU(struct plic_handler, plic_handlers);
  90
  91 static int plic_irq_set_type(struct irq_data *d, unsigned int type);
  92
  93 static void __plic_toggle(void __iomem *enable_base, int hwirq, int enable)
  94 {
  95         u32 __iomem *reg = enable_base + (hwirq / 32) * sizeof(u32);
  96         u32 hwirq_mask = 1 << (hwirq % 32);
  97
  98         if (enable)
  99                 writel(readl(reg) | hwirq_mask, reg);
 100         else
 101                 writel(readl(reg) & ~hwirq_mask, reg);
 102 }
 103
 104 static void plic_toggle(struct plic_handler *handler, int hwirq, int enable)
 105 {
 106         raw_spin_lock(&handler->enable_lock);
 107         __plic_toggle(handler->enable_base, hwirq, enable);
 108         raw_spin_unlock(&handler->enable_lock);
 109 }
 110
 111 static inline void plic_irq_toggle(const struct cpumask *mask,
 112                                    struct irq_data *d, int enable)
 113 {
 114         int cpu;
 115
 116         for_each_cpu(cpu, mask) {
 117                 struct plic_handler *handler = per_cpu_ptr(&plic_handlers, cpu);
 118
 119                 plic_toggle(handler, d->hwirq, enable);
 120         }
 121 }
 122
 123 static void plic_irq_enable(struct irq_data *d)
 124 {
 125         plic_irq_toggle(irq_data_get_effective_affinity_mask(d), d, 1);
 126 }
 127
 128 static void plic_irq_disable(struct irq_data *d)
 129 {
 130         plic_irq_toggle(irq_data_get_effective_affinity_mask(d), d, 0);
 131 }
 132
 133 static void plic_irq_unmask(struct irq_data *d)
 134 {
 135         struct plic_priv *priv = irq_data_get_irq_chip_data(d);
 136
 137         writel(1, priv->regs + PRIORITY_BASE + d->hwirq * PRIORITY_PER_ID);
 138 }
 139
 140 static void plic_irq_mask(struct irq_data *d)
 141 {
 142         struct plic_priv *priv = irq_data_get_irq_chip_data(d);
 143
 144         writel(0, priv->regs + PRIORITY_BASE + d->hwirq * PRIORITY_PER_ID);
 145 }
 146
 147 static void plic_irq_eoi(struct irq_data *d)
 148 {
 149         struct plic_handler *handler = this_cpu_ptr(&plic_handlers);
 150
 151         if (unlikely(irqd_irq_disabled(d))) {
 152                 plic_toggle(handler, d->hwirq, 1);
 153                 writel(d->hwirq, handler->hart_base + CONTEXT_CLAIM);
 154                 plic_toggle(handler, d->hwirq, 0);
 155         } else {
 156                 writel(d->hwirq, handler->hart_base + CONTEXT_CLAIM);
 157         }
 158 }
 159
 160 #ifdef CONFIG_SMP
 161 static int plic_set_affinity(struct irq_data *d,
 162                              const struct cpumask *mask_val, bool force)
 163 {
 164         unsigned int cpu;
 165         struct cpumask amask;
 166         struct plic_priv *priv = irq_data_get_irq_chip_data(d);
 167
 168         cpumask_and(&amask, &priv->lmask, mask_val);
 169
 170         if (force)
 171                 cpu = cpumask_first(&amask);
 172         else
 173                 cpu = cpumask_any_and(&amask, cpu_online_mask);
 174
 175         if (cpu >= nr_cpu_ids)
 176                 return -EINVAL;
 177
 178         plic_irq_disable(d);
 179
 180         irq_data_update_effective_affinity(d, cpumask_of(cpu));
 181
 182         if (!irqd_irq_disabled(d))
 183                 plic_irq_enable(d);
 184
 185         return IRQ_SET_MASK_OK_DONE;
 186 }
 187 #endif
 188
 189 static struct irq_chip plic_edge_chip = {
 190         .name           = "SiFive PLIC",
 191         .irq_enable     = plic_irq_enable,
 192         .irq_disable    = plic_irq_disable,
 193         .irq_ack        = plic_irq_eoi,
 194         .irq_mask       = plic_irq_mask,
 195         .irq_unmask     = plic_irq_unmask,
 196 #ifdef CONFIG_SMP
 197         .irq_set_affinity = plic_set_affinity,
 198 #endif
 199         .irq_set_type   = plic_irq_set_type,
 200         .flags          = IRQCHIP_SKIP_SET_WAKE |
 201                           IRQCHIP_AFFINITY_PRE_STARTUP,
 202 };
 203
 204 static struct irq_chip plic_chip = {
 205         .name           = "SiFive PLIC",
 206         .irq_enable     = plic_irq_enable,
 207         .irq_disable    = plic_irq_disable,
 208         .irq_mask       = plic_irq_mask,
 209         .irq_unmask     = plic_irq_unmask,
 210         .irq_eoi        = plic_irq_eoi,
 211 #ifdef CONFIG_SMP
 212         .irq_set_affinity = plic_set_affinity,
 213 #endif
 214         .irq_set_type   = plic_irq_set_type,
 215         .flags          = IRQCHIP_SKIP_SET_WAKE |
 216                           IRQCHIP_AFFINITY_PRE_STARTUP,
 217 };
 218
 219 static int plic_irq_set_type(struct irq_data *d, unsigned int type)
 220 {
 221         struct plic_priv *priv = irq_data_get_irq_chip_data(d);
 222
 223         if (!test_bit(PLIC_QUIRK_EDGE_INTERRUPT, &priv->plic_quirks))
 224                 return IRQ_SET_MASK_OK_NOCOPY;
 225
 226         switch (type) {
 227         case IRQ_TYPE_EDGE_RISING:
 228                 irq_set_chip_handler_name_locked(d, &plic_edge_chip,
 229                                                  handle_edge_irq, NULL);
 230                 break;
 231         case IRQ_TYPE_LEVEL_HIGH:
 232                 irq_set_chip_handler_name_locked(d, &plic_chip,
 233                                                  handle_fasteoi_irq, NULL);
 234                 break;
 235         default:
 236                 return -EINVAL;
 237         }
 238
 239         return IRQ_SET_MASK_OK;
 240 }
 241
 242 static int plic_irq_suspend(void)
 243 {
 244         unsigned int i, cpu;
 245         u32 __iomem *reg;
 246         struct plic_priv *priv;
 247
 248         priv = per_cpu_ptr(&plic_handlers, smp_processor_id())->priv;
 249
 250         for (i = 0; i < priv->nr_irqs; i++)
 251                 if (readl(priv->regs + PRIORITY_BASE + i * PRIORITY_PER_ID))
 252                         __set_bit(i, priv->prio_save);
 253                 else
 254                         __clear_bit(i, priv->prio_save);
 255
 256         for_each_cpu(cpu, cpu_present_mask) {
 257                 struct plic_handler *handler = per_cpu_ptr(&plic_handlers, cpu);
 258
 259                 if (!handler->present)
 260                         continue;
 261
 262                 raw_spin_lock(&handler->enable_lock);
 263                 for (i = 0; i < DIV_ROUND_UP(priv->nr_irqs, 32); i++) {
 264                         reg = handler->enable_base + i * sizeof(u32);
 265                         handler->enable_save[i] = readl(reg);
 266                 }
 267                 raw_spin_unlock(&handler->enable_lock);
 268         }
 269
 270         return 0;
 271 }
 272
 273 static void plic_irq_resume(void)
 274 {
 275         unsigned int i, index, cpu;
 276         u32 __iomem *reg;
 277         struct plic_priv *priv;
 278
 279         priv = per_cpu_ptr(&plic_handlers, smp_processor_id())->priv;
 280
 281         for (i = 0; i < priv->nr_irqs; i++) {
 282                 index = BIT_WORD(i);
 283                 writel((priv->prio_save[index] & BIT_MASK(i)) ? 1 : 0,
 284                        priv->regs + PRIORITY_BASE + i * PRIORITY_PER_ID);
 285         }
 286
 287         for_each_cpu(cpu, cpu_present_mask) {
 288                 struct plic_handler *handler = per_cpu_ptr(&plic_handlers, cpu);
 289
 290                 if (!handler->present)
 291                         continue;
 292
 293                 raw_spin_lock(&handler->enable_lock);
 294                 for (i = 0; i < DIV_ROUND_UP(priv->nr_irqs, 32); i++) {
 295                         reg = handler->enable_base + i * sizeof(u32);
 296                         writel(handler->enable_save[i], reg);
 297                 }
 298                 raw_spin_unlock(&handler->enable_lock);
 299         }
 300 }
 301
 302 static struct syscore_ops plic_irq_syscore_ops = {
 303         .suspend        = plic_irq_suspend,
 304         .resume         = plic_irq_resume,
 305 };
 306
 307 static int plic_irqdomain_map(struct irq_domain *d, unsigned int irq,
 308                               irq_hw_number_t hwirq)
 309 {
 310         struct plic_priv *priv = d->host_data;
 311
 312         irq_domain_set_info(d, irq, hwirq, &plic_chip, d->host_data,
 313                             handle_fasteoi_irq, NULL, NULL);
 314         irq_set_noprobe(irq);
 315         irq_set_affinity(irq, &priv->lmask);
 316         return 0;
 317 }
 318
 319 static int plic_irq_domain_translate(struct irq_domain *d,
 320                                      struct irq_fwspec *fwspec,
 321                                      unsigned long *hwirq,
 322                                      unsigned int *type)
 323 {
 324         struct plic_priv *priv = d->host_data;
 325
 326         if (test_bit(PLIC_QUIRK_EDGE_INTERRUPT, &priv->plic_quirks))
 327                 return irq_domain_translate_twocell(d, fwspec, hwirq, type);
 328
 329         return irq_domain_translate_onecell(d, fwspec, hwirq, type);
 330 }
 331
 332 static int plic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
 333                                  unsigned int nr_irqs, void *arg)
 334 {
 335         int i, ret;
 336         irq_hw_number_t hwirq;
 337         unsigned int type;
 338         struct irq_fwspec *fwspec = arg;
 339
 340         ret = plic_irq_domain_translate(domain, fwspec, &hwirq, &type);
 341         if (ret)
 342                 return ret;
 343
 344         for (i = 0; i < nr_irqs; i++) {
 345                 ret = plic_irqdomain_map(domain, virq + i, hwirq + i);
 346                 if (ret)
 347                         return ret;
 348         }
 349
 350         return 0;
 351 }
 352
 353 static const struct irq_domain_ops plic_irqdomain_ops = {
 354         .translate      = plic_irq_domain_translate,
 355         .alloc          = plic_irq_domain_alloc,
 356         .free           = irq_domain_free_irqs_top,
 357 };
 358
 359 /*
 360  * Handling an interrupt is a two-step process: first you claim the interrupt
 361  * by reading the claim register, then you complete the interrupt by writing
 362  * that source ID back to the same claim register.  This automatically enables
 363  * and disables the interrupt, so there's nothing else to do.
 364  */
 365 static void plic_handle_irq(struct irq_desc *desc)
 366 {
 367         struct plic_handler *handler = this_cpu_ptr(&plic_handlers);
 368         struct irq_chip *chip = irq_desc_get_chip(desc);
 369         void __iomem *claim = handler->hart_base + CONTEXT_CLAIM;
 370         irq_hw_number_t hwirq;
 371
 372         WARN_ON_ONCE(!handler->present);
 373
 374         chained_irq_enter(chip, desc);
 375
 376         while ((hwirq = readl(claim))) {
 377                 int err = generic_handle_domain_irq(handler->priv->irqdomain,
 378                                                     hwirq);
 379                 if (unlikely(err))
 380                         pr_warn_ratelimited("can't find mapping for hwirq %lu\n",
 381                                         hwirq);
 382         }
 383
 384         chained_irq_exit(chip, desc);
 385 }
 386
 387 static void plic_set_threshold(struct plic_handler *handler, u32 threshold)
 388 {
 389         /* priority must be > threshold to trigger an interrupt */
 390         writel(threshold, handler->hart_base + CONTEXT_THRESHOLD);
 391 }
 392
 393 static int plic_dying_cpu(unsigned int cpu)
 394 {
 395         if (plic_parent_irq)
 396                 disable_percpu_irq(plic_parent_irq);
 397
 398         return 0;
 399 }
 400
 401 static int plic_starting_cpu(unsigned int cpu)
 402 {
 403         struct plic_handler *handler = this_cpu_ptr(&plic_handlers);
 404
 405         if (plic_parent_irq)
 406                 enable_percpu_irq(plic_parent_irq,
 407                                   irq_get_trigger_type(plic_parent_irq));
 408         else
 409                 pr_warn("cpu%d: parent irq not available\n", cpu);
 410         plic_set_threshold(handler, PLIC_ENABLE_THRESHOLD);
 411
 412         return 0;
 413 }
 414
 415 static int __init __plic_init(struct device_node *node,
 416                               struct device_node *parent,
 417                               unsigned long plic_quirks)
 418 {
 419         int error = 0, nr_contexts, nr_handlers = 0, i;
 420         u32 nr_irqs;
 421         struct plic_priv *priv;
 422         struct plic_handler *handler;
 423         unsigned int cpu;
 424
 425         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
 426         if (!priv)
 427                 return -ENOMEM;
 428
 429         priv->plic_quirks = plic_quirks;
 430
 431         priv->regs = of_iomap(node, 0);
 432         if (WARN_ON(!priv->regs)) {
 433                 error = -EIO;
 434                 goto out_free_priv;
 435         }
 436
 437         error = -EINVAL;
 438         of_property_read_u32(node, "riscv,ndev", &nr_irqs);
 439         if (WARN_ON(!nr_irqs))
 440                 goto out_iounmap;
 441
 442         priv->nr_irqs = nr_irqs;
 443
 444         priv->prio_save = bitmap_alloc(nr_irqs, GFP_KERNEL);
 445         if (!priv->prio_save)
 446                 goto out_free_priority_reg;
 447
 448         nr_contexts = of_irq_count(node);
 449         if (WARN_ON(!nr_contexts))
 450                 goto out_free_priority_reg;
 451
 452         error = -ENOMEM;
 453         priv->irqdomain = irq_domain_add_linear(node, nr_irqs + 1,
 454                         &plic_irqdomain_ops, priv);
 455         if (WARN_ON(!priv->irqdomain))
 456                 goto out_free_priority_reg;
 457
 458         for (i = 0; i < nr_contexts; i++) {
 459                 struct of_phandle_args parent;
 460                 irq_hw_number_t hwirq;
 461                 int cpu;
 462                 unsigned long hartid;
 463
 464                 if (of_irq_parse_one(node, i, &parent)) {
 465                         pr_err("failed to parse parent for context %d.\n", i);
 466                         continue;
 467                 }
 468
 469                 /*
 470                  * Skip contexts other than external interrupts for our
 471                  * privilege level.
 472                  */
 473                 if (parent.args[0] != RV_IRQ_EXT) {
 474                         /* Disable S-mode enable bits if running in M-mode. */
 475                         if (IS_ENABLED(CONFIG_RISCV_M_MODE)) {
 476                                 void __iomem *enable_base = priv->regs +
 477                                         CONTEXT_ENABLE_BASE +
 478                                         i * CONTEXT_ENABLE_SIZE;
 479
 480                                 for (hwirq = 1; hwirq <= nr_irqs; hwirq++)
 481                                         __plic_toggle(enable_base, hwirq, 0);
 482                         }
 483                         continue;
 484                 }
 485
 486                 error = riscv_of_parent_hartid(parent.np, &hartid);
 487                 if (error < 0) {
 488                         pr_warn("failed to parse hart ID for context %d.\n", i);
 489                         continue;
 490                 }
 491
 492                 cpu = riscv_hartid_to_cpuid(hartid);
 493                 if (cpu < 0) {
 494                         pr_warn("Invalid cpuid for context %d\n", i);
 495                         continue;
 496                 }
 497
 498                 /* Find parent domain and register chained handler */
 499                 if (!plic_parent_irq && irq_find_host(parent.np)) {
 500                         plic_parent_irq = irq_of_parse_and_map(node, i);
 501                         if (plic_parent_irq)
 502                                 irq_set_chained_handler(plic_parent_irq,
 503                                                         plic_handle_irq);
 504                 }
 505
 506                 /*
 507                  * When running in M-mode we need to ignore the S-mode handler.
 508                  * Here we assume it always comes later, but that might be a
 509                  * little fragile.
 510                  */
 511                 handler = per_cpu_ptr(&plic_handlers, cpu);
 512                 if (handler->present) {
 513                         pr_warn("handler already present for context %d.\n", i);
 514                         plic_set_threshold(handler, PLIC_DISABLE_THRESHOLD);
 515                         goto done;
 516                 }
 517
 518                 cpumask_set_cpu(cpu, &priv->lmask);
 519                 handler->present = true;
 520                 handler->hart_base = priv->regs + CONTEXT_BASE +
 521                         i * CONTEXT_SIZE;
 522                 raw_spin_lock_init(&handler->enable_lock);
 523                 handler->enable_base = priv->regs + CONTEXT_ENABLE_BASE +
 524                         i * CONTEXT_ENABLE_SIZE;
 525                 handler->priv = priv;
 526
 527                 handler->enable_save =  kcalloc(DIV_ROUND_UP(nr_irqs, 32),
 528                                                 sizeof(*handler->enable_save), GFP_KERNEL);
 529                 if (!handler->enable_save)
 530                         goto out_free_enable_reg;
 531 done:
 532                 for (hwirq = 1; hwirq <= nr_irqs; hwirq++) {
 533                         plic_toggle(handler, hwirq, 0);
 534                         writel(1, priv->regs + PRIORITY_BASE +
 535                                   hwirq * PRIORITY_PER_ID);
 536                 }
 537                 nr_handlers++;
 538         }
 539
 540         /*
 541          * We can have multiple PLIC instances so setup cpuhp state
 542          * and register syscore operations only when context handler
 543          * for current/boot CPU is present.
 544          */
 545         handler = this_cpu_ptr(&plic_handlers);
 546         if (handler->present && !plic_cpuhp_setup_done) {
 547                 cpuhp_setup_state(CPUHP_AP_IRQ_SIFIVE_PLIC_STARTING,
 548                                   "irqchip/sifive/plic:starting",
 549                                   plic_starting_cpu, plic_dying_cpu);
 550                 register_syscore_ops(&plic_irq_syscore_ops);
 551                 plic_cpuhp_setup_done = true;
 552         }
 553
 554         pr_info("%pOFP: mapped %d interrupts with %d handlers for"
 555                 " %d contexts.\n", node, nr_irqs, nr_handlers, nr_contexts);
 556         return 0;
 557
 558 out_free_enable_reg:
 559         for_each_cpu(cpu, cpu_present_mask) {
 560                 handler = per_cpu_ptr(&plic_handlers, cpu);
 561                 kfree(handler->enable_save);
 562         }
 563 out_free_priority_reg:
 564         kfree(priv->prio_save);
 565 out_iounmap:
 566         iounmap(priv->regs);
 567 out_free_priv:
 568         kfree(priv);
 569         return error;
 570 }
 571
 572 static int __init plic_init(struct device_node *node,
 573                             struct device_node *parent)
 574 {
 575         return __plic_init(node, parent, 0);
 576 }
 577
 578 IRQCHIP_DECLARE(sifive_plic, "sifive,plic-1.0.0", plic_init);
 579 IRQCHIP_DECLARE(riscv_plic0, "riscv,plic0", plic_init); /* for legacy systems */
 580
 581 static int __init plic_edge_init(struct device_node *node,
 582                                  struct device_node *parent)
 583 {
 584         return __plic_init(node, parent, BIT(PLIC_QUIRK_EDGE_INTERRUPT));
 585 }
 586
 587 IRQCHIP_DECLARE(andestech_nceplic100, "andestech,nceplic100", plic_edge_init);
 588 IRQCHIP_DECLARE(thead_c900_plic, "thead,c900-plic", plic_edge_init);