drivers/irqchip/irq-mips-gic.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 2008 Ralf Baechle (ralf@linux-mips.org)
   7  * Copyright (C) 2012 MIPS Technologies, Inc.  All rights reserved.
   8  */
   9 #include <linux/bitmap.h>
  10 #include <linux/clocksource.h>
  11 #include <linux/init.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/irq.h>
  14 #include <linux/irqchip.h>
  15 #include <linux/irqchip/mips-gic.h>
  16 #include <linux/of_address.h>
  17 #include <linux/sched.h>
  18 #include <linux/smp.h>
  19
  20 #include <asm/mips-cm.h>
  21 #include <asm/setup.h>
  22 #include <asm/traps.h>
  23
  24 #include <dt-bindings/interrupt-controller/mips-gic.h>
  25
  26 unsigned int gic_present;
  27
  28 struct gic_pcpu_mask {
  29         DECLARE_BITMAP(pcpu_mask, GIC_MAX_INTRS);
  30 };
  31
  32 struct gic_irq_spec {
  33         enum {
  34                 GIC_DEVICE,
  35                 GIC_IPI
  36         } type;
  37
  38         union {
  39                 struct cpumask *ipimask;
  40                 unsigned int hwirq;
  41         };
  42 };
  43
  44 static unsigned long __gic_base_addr;
  45
  46 static void __iomem *gic_base;
  47 static struct gic_pcpu_mask pcpu_masks[NR_CPUS];
  48 static DEFINE_SPINLOCK(gic_lock);
  49 static struct irq_domain *gic_irq_domain;
  50 static struct irq_domain *gic_dev_domain;
  51 static struct irq_domain *gic_ipi_domain;
  52 static int gic_shared_intrs;
  53 static int gic_vpes;
  54 static unsigned int gic_cpu_pin;
  55 static unsigned int timer_cpu_pin;
  56 static struct irq_chip gic_level_irq_controller, gic_edge_irq_controller;
  57 DECLARE_BITMAP(ipi_resrv, GIC_MAX_INTRS);
  58 DECLARE_BITMAP(ipi_available, GIC_MAX_INTRS);
  59
  60 static void __gic_irq_dispatch(void);
  61
  62 static inline u32 gic_read32(unsigned int reg)
  63 {
  64         return __raw_readl(gic_base + reg);
  65 }
  66
  67 static inline u64 gic_read64(unsigned int reg)
  68 {
  69         return __raw_readq(gic_base + reg);
  70 }
  71
  72 static inline unsigned long gic_read(unsigned int reg)
  73 {
  74         if (!mips_cm_is64)
  75                 return gic_read32(reg);
  76         else
  77                 return gic_read64(reg);
  78 }
  79
  80 static inline void gic_write32(unsigned int reg, u32 val)
  81 {
  82         return __raw_writel(val, gic_base + reg);
  83 }
  84
  85 static inline void gic_write64(unsigned int reg, u64 val)
  86 {
  87         return __raw_writeq(val, gic_base + reg);
  88 }
  89
  90 static inline void gic_write(unsigned int reg, unsigned long val)
  91 {
  92         if (!mips_cm_is64)
  93                 return gic_write32(reg, (u32)val);
  94         else
  95                 return gic_write64(reg, (u64)val);
  96 }
  97
  98 static inline void gic_update_bits(unsigned int reg, unsigned long mask,
  99                                    unsigned long val)
 100 {
 101         unsigned long regval;
 102
 103         regval = gic_read(reg);
 104         regval &= ~mask;
 105         regval |= val;
 106         gic_write(reg, regval);
 107 }
 108
 109 static inline void gic_reset_mask(unsigned int intr)
 110 {
 111         gic_write(GIC_REG(SHARED, GIC_SH_RMASK) + GIC_INTR_OFS(intr),
 112                   1ul << GIC_INTR_BIT(intr));
 113 }
 114
 115 static inline void gic_set_mask(unsigned int intr)
 116 {
 117         gic_write(GIC_REG(SHARED, GIC_SH_SMASK) + GIC_INTR_OFS(intr),
 118                   1ul << GIC_INTR_BIT(intr));
 119 }
 120
 121 static inline void gic_set_polarity(unsigned int intr, unsigned int pol)
 122 {
 123         gic_update_bits(GIC_REG(SHARED, GIC_SH_SET_POLARITY) +
 124                         GIC_INTR_OFS(intr), 1ul << GIC_INTR_BIT(intr),
 125                         (unsigned long)pol << GIC_INTR_BIT(intr));
 126 }
 127
 128 static inline void gic_set_trigger(unsigned int intr, unsigned int trig)
 129 {
 130         gic_update_bits(GIC_REG(SHARED, GIC_SH_SET_TRIGGER) +
 131                         GIC_INTR_OFS(intr), 1ul << GIC_INTR_BIT(intr),
 132                         (unsigned long)trig << GIC_INTR_BIT(intr));
 133 }
 134
 135 static inline void gic_set_dual_edge(unsigned int intr, unsigned int dual)
 136 {
 137         gic_update_bits(GIC_REG(SHARED, GIC_SH_SET_DUAL) + GIC_INTR_OFS(intr),
 138                         1ul << GIC_INTR_BIT(intr),
 139                         (unsigned long)dual << GIC_INTR_BIT(intr));
 140 }
 141
 142 static inline void gic_map_to_pin(unsigned int intr, unsigned int pin)
 143 {
 144         gic_write32(GIC_REG(SHARED, GIC_SH_INTR_MAP_TO_PIN_BASE) +
 145                     GIC_SH_MAP_TO_PIN(intr), GIC_MAP_TO_PIN_MSK | pin);
 146 }
 147
 148 static inline void gic_map_to_vpe(unsigned int intr, unsigned int vpe)
 149 {
 150         gic_write(GIC_REG(SHARED, GIC_SH_INTR_MAP_TO_VPE_BASE) +
 151                   GIC_SH_MAP_TO_VPE_REG_OFF(intr, vpe),
 152                   GIC_SH_MAP_TO_VPE_REG_BIT(vpe));
 153 }
 154
 155 #ifdef CONFIG_CLKSRC_MIPS_GIC
 156 cycle_t gic_read_count(void)
 157 {
 158         unsigned int hi, hi2, lo;
 159
 160         if (mips_cm_is64)
 161                 return (cycle_t)gic_read(GIC_REG(SHARED, GIC_SH_COUNTER));
 162
 163         do {
 164                 hi = gic_read32(GIC_REG(SHARED, GIC_SH_COUNTER_63_32));
 165                 lo = gic_read32(GIC_REG(SHARED, GIC_SH_COUNTER_31_00));
 166                 hi2 = gic_read32(GIC_REG(SHARED, GIC_SH_COUNTER_63_32));
 167         } while (hi2 != hi);
 168
 169         return (((cycle_t) hi) << 32) + lo;
 170 }
 171
 172 unsigned int gic_get_count_width(void)
 173 {
 174         unsigned int bits, config;
 175
 176         config = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
 177         bits = 32 + 4 * ((config & GIC_SH_CONFIG_COUNTBITS_MSK) >>
 178                          GIC_SH_CONFIG_COUNTBITS_SHF);
 179
 180         return bits;
 181 }
 182
 183 void gic_write_compare(cycle_t cnt)
 184 {
 185         if (mips_cm_is64) {
 186                 gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE), cnt);
 187         } else {
 188                 gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_HI),
 189                                         (int)(cnt >> 32));
 190                 gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_LO),
 191                                         (int)(cnt & 0xffffffff));
 192         }
 193 }
 194
 195 void gic_write_cpu_compare(cycle_t cnt, int cpu)
 196 {
 197         unsigned long flags;
 198
 199         local_irq_save(flags);
 200
 201         gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR), mips_cm_vp_id(cpu));
 202
 203         if (mips_cm_is64) {
 204                 gic_write(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE), cnt);
 205         } else {
 206                 gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE_HI),
 207                                         (int)(cnt >> 32));
 208                 gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE_LO),
 209                                         (int)(cnt & 0xffffffff));
 210         }
 211
 212         local_irq_restore(flags);
 213 }
 214
 215 cycle_t gic_read_compare(void)
 216 {
 217         unsigned int hi, lo;
 218
 219         if (mips_cm_is64)
 220                 return (cycle_t)gic_read(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE));
 221
 222         hi = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_HI));
 223         lo = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_LO));
 224
 225         return (((cycle_t) hi) << 32) + lo;
 226 }
 227
 228 void gic_start_count(void)
 229 {
 230         u32 gicconfig;
 231
 232         /* Start the counter */
 233         gicconfig = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
 234         gicconfig &= ~(1 << GIC_SH_CONFIG_COUNTSTOP_SHF);
 235         gic_write(GIC_REG(SHARED, GIC_SH_CONFIG), gicconfig);
 236 }
 237
 238 void gic_stop_count(void)
 239 {
 240         u32 gicconfig;
 241
 242         /* Stop the counter */
 243         gicconfig = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
 244         gicconfig |= 1 << GIC_SH_CONFIG_COUNTSTOP_SHF;
 245         gic_write(GIC_REG(SHARED, GIC_SH_CONFIG), gicconfig);
 246 }
 247
 248 #endif
 249
 250 unsigned gic_read_local_vp_id(void)
 251 {
 252         unsigned long ident;
 253
 254         ident = gic_read(GIC_REG(VPE_LOCAL, GIC_VP_IDENT));
 255         return ident & GIC_VP_IDENT_VCNUM_MSK;
 256 }
 257
 258 static bool gic_local_irq_is_routable(int intr)
 259 {
 260         u32 vpe_ctl;
 261
 262         /* All local interrupts are routable in EIC mode. */
 263         if (cpu_has_veic)
 264                 return true;
 265
 266         vpe_ctl = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_CTL));
 267         switch (intr) {
 268         case GIC_LOCAL_INT_TIMER:
 269                 return vpe_ctl & GIC_VPE_CTL_TIMER_RTBL_MSK;
 270         case GIC_LOCAL_INT_PERFCTR:
 271                 return vpe_ctl & GIC_VPE_CTL_PERFCNT_RTBL_MSK;
 272         case GIC_LOCAL_INT_FDC:
 273                 return vpe_ctl & GIC_VPE_CTL_FDC_RTBL_MSK;
 274         case GIC_LOCAL_INT_SWINT0:
 275         case GIC_LOCAL_INT_SWINT1:
 276                 return vpe_ctl & GIC_VPE_CTL_SWINT_RTBL_MSK;
 277         default:
 278                 return true;
 279         }
 280 }
 281
 282 static void gic_bind_eic_interrupt(int irq, int set)
 283 {
 284         /* Convert irq vector # to hw int # */
 285         irq -= GIC_PIN_TO_VEC_OFFSET;
 286
 287         /* Set irq to use shadow set */
 288         gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_EIC_SHADOW_SET_BASE) +
 289                   GIC_VPE_EIC_SS(irq), set);
 290 }
 291
 292 static void gic_send_ipi(struct irq_data *d, unsigned int cpu)
 293 {
 294         irq_hw_number_t hwirq = GIC_HWIRQ_TO_SHARED(irqd_to_hwirq(d));
 295
 296         gic_write(GIC_REG(SHARED, GIC_SH_WEDGE), GIC_SH_WEDGE_SET(hwirq));
 297 }
 298
 299 int gic_get_c0_compare_int(void)
 300 {
 301         if (!gic_local_irq_is_routable(GIC_LOCAL_INT_TIMER))
 302                 return MIPS_CPU_IRQ_BASE + cp0_compare_irq;
 303         return irq_create_mapping(gic_irq_domain,
 304                                   GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_TIMER));
 305 }
 306
 307 int gic_get_c0_perfcount_int(void)
 308 {
 309         if (!gic_local_irq_is_routable(GIC_LOCAL_INT_PERFCTR)) {
 310                 /* Is the performance counter shared with the timer? */
 311                 if (cp0_perfcount_irq < 0)
 312                         return -1;
 313                 return MIPS_CPU_IRQ_BASE + cp0_perfcount_irq;
 314         }
 315         return irq_create_mapping(gic_irq_domain,
 316                                   GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_PERFCTR));
 317 }
 318
 319 int gic_get_c0_fdc_int(void)
 320 {
 321         if (!gic_local_irq_is_routable(GIC_LOCAL_INT_FDC)) {
 322                 /* Is the FDC IRQ even present? */
 323                 if (cp0_fdc_irq < 0)
 324                         return -1;
 325                 return MIPS_CPU_IRQ_BASE + cp0_fdc_irq;
 326         }
 327
 328         return irq_create_mapping(gic_irq_domain,
 329                                   GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_FDC));
 330 }
 331
 332 int gic_get_usm_range(struct resource *gic_usm_res)
 333 {
 334         if (!gic_present)
 335                 return -1;
 336
 337         gic_usm_res->start = __gic_base_addr + USM_VISIBLE_SECTION_OFS;
 338         gic_usm_res->end = gic_usm_res->start + (USM_VISIBLE_SECTION_SIZE - 1);
 339
 340         return 0;
 341 }
 342
 343 static void gic_handle_shared_int(bool chained)
 344 {
 345         unsigned int i, intr, virq, gic_reg_step = mips_cm_is64 ? 8 : 4;
 346         unsigned long *pcpu_mask;
 347         unsigned long pending_reg, intrmask_reg;
 348         DECLARE_BITMAP(pending, GIC_MAX_INTRS);
 349         DECLARE_BITMAP(intrmask, GIC_MAX_INTRS);
 350
 351         /* Get per-cpu bitmaps */
 352         pcpu_mask = pcpu_masks[smp_processor_id()].pcpu_mask;
 353
 354         pending_reg = GIC_REG(SHARED, GIC_SH_PEND);
 355         intrmask_reg = GIC_REG(SHARED, GIC_SH_MASK);
 356
 357         for (i = 0; i < BITS_TO_LONGS(gic_shared_intrs); i++) {
 358                 pending[i] = gic_read(pending_reg);
 359                 intrmask[i] = gic_read(intrmask_reg);
 360                 pending_reg += gic_reg_step;
 361                 intrmask_reg += gic_reg_step;
 362
 363                 if (!IS_ENABLED(CONFIG_64BIT) || mips_cm_is64)
 364                         continue;
 365
 366                 pending[i] |= (u64)gic_read(pending_reg) << 32;
 367                 intrmask[i] |= (u64)gic_read(intrmask_reg) << 32;
 368                 pending_reg += gic_reg_step;
 369                 intrmask_reg += gic_reg_step;
 370         }
 371
 372         bitmap_and(pending, pending, intrmask, gic_shared_intrs);
 373         bitmap_and(pending, pending, pcpu_mask, gic_shared_intrs);
 374
 375         for_each_set_bit(intr, pending, gic_shared_intrs) {
 376                 virq = irq_linear_revmap(gic_irq_domain,
 377                                          GIC_SHARED_TO_HWIRQ(intr));
 378                 if (chained)
 379                         generic_handle_irq(virq);
 380                 else
 381                         do_IRQ(virq);
 382         }
 383 }
 384
 385 static void gic_mask_irq(struct irq_data *d)
 386 {
 387         gic_reset_mask(GIC_HWIRQ_TO_SHARED(d->hwirq));
 388 }
 389
 390 static void gic_unmask_irq(struct irq_data *d)
 391 {
 392         gic_set_mask(GIC_HWIRQ_TO_SHARED(d->hwirq));
 393 }
 394
 395 static void gic_ack_irq(struct irq_data *d)
 396 {
 397         unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
 398
 399         gic_write(GIC_REG(SHARED, GIC_SH_WEDGE), GIC_SH_WEDGE_CLR(irq));
 400 }
 401
 402 static int gic_set_type(struct irq_data *d, unsigned int type)
 403 {
 404         unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
 405         unsigned long flags;
 406         bool is_edge;
 407
 408         spin_lock_irqsave(&gic_lock, flags);
 409         switch (type & IRQ_TYPE_SENSE_MASK) {
 410         case IRQ_TYPE_EDGE_FALLING:
 411                 gic_set_polarity(irq, GIC_POL_NEG);
 412                 gic_set_trigger(irq, GIC_TRIG_EDGE);
 413                 gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
 414                 is_edge = true;
 415                 break;
 416         case IRQ_TYPE_EDGE_RISING:
 417                 gic_set_polarity(irq, GIC_POL_POS);
 418                 gic_set_trigger(irq, GIC_TRIG_EDGE);
 419                 gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
 420                 is_edge = true;
 421                 break;
 422         case IRQ_TYPE_EDGE_BOTH:
 423                 /* polarity is irrelevant in this case */
 424                 gic_set_trigger(irq, GIC_TRIG_EDGE);
 425                 gic_set_dual_edge(irq, GIC_TRIG_DUAL_ENABLE);
 426                 is_edge = true;
 427                 break;
 428         case IRQ_TYPE_LEVEL_LOW:
 429                 gic_set_polarity(irq, GIC_POL_NEG);
 430                 gic_set_trigger(irq, GIC_TRIG_LEVEL);
 431                 gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
 432                 is_edge = false;
 433                 break;
 434         case IRQ_TYPE_LEVEL_HIGH:
 435         default:
 436                 gic_set_polarity(irq, GIC_POL_POS);
 437                 gic_set_trigger(irq, GIC_TRIG_LEVEL);
 438                 gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
 439                 is_edge = false;
 440                 break;
 441         }
 442
 443         if (is_edge)
 444                 irq_set_chip_handler_name_locked(d, &gic_edge_irq_controller,
 445                                                  handle_edge_irq, NULL);
 446         else
 447                 irq_set_chip_handler_name_locked(d, &gic_level_irq_controller,
 448                                                  handle_level_irq, NULL);
 449         spin_unlock_irqrestore(&gic_lock, flags);
 450
 451         return 0;
 452 }
 453
 454 #ifdef CONFIG_SMP
 455 static int gic_set_affinity(struct irq_data *d, const struct cpumask *cpumask,
 456                             bool force)
 457 {
 458         unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
 459         cpumask_t       tmp = CPU_MASK_NONE;
 460         unsigned long   flags;
 461         int             i;
 462
 463         cpumask_and(&tmp, cpumask, cpu_online_mask);
 464         if (cpumask_empty(&tmp))
 465                 return -EINVAL;
 466
 467         /* Assumption : cpumask refers to a single CPU */
 468         spin_lock_irqsave(&gic_lock, flags);
 469
 470         /* Re-route this IRQ */
 471         gic_map_to_vpe(irq, mips_cm_vp_id(cpumask_first(&tmp)));
 472
 473         /* Update the pcpu_masks */
 474         for (i = 0; i < min(gic_vpes, NR_CPUS); i++)
 475                 clear_bit(irq, pcpu_masks[i].pcpu_mask);
 476         set_bit(irq, pcpu_masks[cpumask_first(&tmp)].pcpu_mask);
 477
 478         cpumask_copy(irq_data_get_affinity_mask(d), cpumask);
 479         spin_unlock_irqrestore(&gic_lock, flags);
 480
 481         return IRQ_SET_MASK_OK_NOCOPY;
 482 }
 483 #endif
 484
 485 static struct irq_chip gic_level_irq_controller = {
 486         .name                   =       "MIPS GIC",
 487         .irq_mask               =       gic_mask_irq,
 488         .irq_unmask             =       gic_unmask_irq,
 489         .irq_set_type           =       gic_set_type,
 490 #ifdef CONFIG_SMP
 491         .irq_set_affinity       =       gic_set_affinity,
 492 #endif
 493 };
 494
 495 static struct irq_chip gic_edge_irq_controller = {
 496         .name                   =       "MIPS GIC",
 497         .irq_ack                =       gic_ack_irq,
 498         .irq_mask               =       gic_mask_irq,
 499         .irq_unmask             =       gic_unmask_irq,
 500         .irq_set_type           =       gic_set_type,
 501 #ifdef CONFIG_SMP
 502         .irq_set_affinity       =       gic_set_affinity,
 503 #endif
 504         .ipi_send_single        =       gic_send_ipi,
 505 };
 506
 507 static void gic_handle_local_int(bool chained)
 508 {
 509         unsigned long pending, masked;
 510         unsigned int intr, virq;
 511
 512         pending = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_PEND));
 513         masked = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_MASK));
 514
 515         bitmap_and(&pending, &pending, &masked, GIC_NUM_LOCAL_INTRS);
 516
 517         for_each_set_bit(intr, &pending, GIC_NUM_LOCAL_INTRS) {
 518                 virq = irq_linear_revmap(gic_irq_domain,
 519                                          GIC_LOCAL_TO_HWIRQ(intr));
 520                 if (chained)
 521                         generic_handle_irq(virq);
 522                 else
 523                         do_IRQ(virq);
 524         }
 525 }
 526
 527 static void gic_mask_local_irq(struct irq_data *d)
 528 {
 529         int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
 530
 531         gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_RMASK), 1 << intr);
 532 }
 533
 534 static void gic_unmask_local_irq(struct irq_data *d)
 535 {
 536         int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
 537
 538         gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_SMASK), 1 << intr);
 539 }
 540
 541 static struct irq_chip gic_local_irq_controller = {
 542         .name                   =       "MIPS GIC Local",
 543         .irq_mask               =       gic_mask_local_irq,
 544         .irq_unmask             =       gic_unmask_local_irq,
 545 };
 546
 547 static void gic_mask_local_irq_all_vpes(struct irq_data *d)
 548 {
 549         int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
 550         int i;
 551         unsigned long flags;
 552
 553         spin_lock_irqsave(&gic_lock, flags);
 554         for (i = 0; i < gic_vpes; i++) {
 555                 gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
 556                           mips_cm_vp_id(i));
 557                 gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_RMASK), 1 << intr);
 558         }
 559         spin_unlock_irqrestore(&gic_lock, flags);
 560 }
 561
 562 static void gic_unmask_local_irq_all_vpes(struct irq_data *d)
 563 {
 564         int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
 565         int i;
 566         unsigned long flags;
 567
 568         spin_lock_irqsave(&gic_lock, flags);
 569         for (i = 0; i < gic_vpes; i++) {
 570                 gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
 571                           mips_cm_vp_id(i));
 572                 gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_SMASK), 1 << intr);
 573         }
 574         spin_unlock_irqrestore(&gic_lock, flags);
 575 }
 576
 577 static struct irq_chip gic_all_vpes_local_irq_controller = {
 578         .name                   =       "MIPS GIC Local",
 579         .irq_mask               =       gic_mask_local_irq_all_vpes,
 580         .irq_unmask             =       gic_unmask_local_irq_all_vpes,
 581 };
 582
 583 static void __gic_irq_dispatch(void)
 584 {
 585         gic_handle_local_int(false);
 586         gic_handle_shared_int(false);
 587 }
 588
 589 static void gic_irq_dispatch(struct irq_desc *desc)
 590 {
 591         gic_handle_local_int(true);
 592         gic_handle_shared_int(true);
 593 }
 594
 595 static void __init gic_basic_init(void)
 596 {
 597         unsigned int i;
 598
 599         board_bind_eic_interrupt = &gic_bind_eic_interrupt;
 600
 601         /* Setup defaults */
 602         for (i = 0; i < gic_shared_intrs; i++) {
 603                 gic_set_polarity(i, GIC_POL_POS);
 604                 gic_set_trigger(i, GIC_TRIG_LEVEL);
 605                 gic_reset_mask(i);
 606         }
 607
 608         for (i = 0; i < gic_vpes; i++) {
 609                 unsigned int j;
 610
 611                 gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
 612                           mips_cm_vp_id(i));
 613                 for (j = 0; j < GIC_NUM_LOCAL_INTRS; j++) {
 614                         if (!gic_local_irq_is_routable(j))
 615                                 continue;
 616                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_RMASK), 1 << j);
 617                 }
 618         }
 619 }
 620
 621 static int gic_local_irq_domain_map(struct irq_domain *d, unsigned int virq,
 622                                     irq_hw_number_t hw)
 623 {
 624         int intr = GIC_HWIRQ_TO_LOCAL(hw);
 625         int ret = 0;
 626         int i;
 627         unsigned long flags;
 628
 629         if (!gic_local_irq_is_routable(intr))
 630                 return -EPERM;
 631
 632         spin_lock_irqsave(&gic_lock, flags);
 633         for (i = 0; i < gic_vpes; i++) {
 634                 u32 val = GIC_MAP_TO_PIN_MSK | gic_cpu_pin;
 635
 636                 gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
 637                           mips_cm_vp_id(i));
 638
 639                 switch (intr) {
 640                 case GIC_LOCAL_INT_WD:
 641                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_WD_MAP), val);
 642                         break;
 643                 case GIC_LOCAL_INT_COMPARE:
 644                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE_MAP),
 645                                     val);
 646                         break;
 647                 case GIC_LOCAL_INT_TIMER:
 648                         /* CONFIG_MIPS_CMP workaround (see __gic_init) */
 649                         val = GIC_MAP_TO_PIN_MSK | timer_cpu_pin;
 650                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_TIMER_MAP),
 651                                     val);
 652                         break;
 653                 case GIC_LOCAL_INT_PERFCTR:
 654                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_PERFCTR_MAP),
 655                                     val);
 656                         break;
 657                 case GIC_LOCAL_INT_SWINT0:
 658                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_SWINT0_MAP),
 659                                     val);
 660                         break;
 661                 case GIC_LOCAL_INT_SWINT1:
 662                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_SWINT1_MAP),
 663                                     val);
 664                         break;
 665                 case GIC_LOCAL_INT_FDC:
 666                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_FDC_MAP), val);
 667                         break;
 668                 default:
 669                         pr_err("Invalid local IRQ %d\n", intr);
 670                         ret = -EINVAL;
 671                         break;
 672                 }
 673         }
 674         spin_unlock_irqrestore(&gic_lock, flags);
 675
 676         return ret;
 677 }
 678
 679 static int gic_shared_irq_domain_map(struct irq_domain *d, unsigned int virq,
 680                                      irq_hw_number_t hw, unsigned int vpe)
 681 {
 682         int intr = GIC_HWIRQ_TO_SHARED(hw);
 683         unsigned long flags;
 684         int i;
 685
 686         spin_lock_irqsave(&gic_lock, flags);
 687         gic_map_to_pin(intr, gic_cpu_pin);
 688         gic_map_to_vpe(intr, mips_cm_vp_id(vpe));
 689         for (i = 0; i < min(gic_vpes, NR_CPUS); i++)
 690                 clear_bit(intr, pcpu_masks[i].pcpu_mask);
 691         set_bit(intr, pcpu_masks[vpe].pcpu_mask);
 692         spin_unlock_irqrestore(&gic_lock, flags);
 693
 694         return 0;
 695 }
 696
 697 static int gic_setup_dev_chip(struct irq_domain *d, unsigned int virq,
 698                               unsigned int hwirq)
 699 {
 700         struct irq_chip *chip;
 701         int err;
 702
 703         if (hwirq >= GIC_SHARED_HWIRQ_BASE) {
 704                 err = irq_domain_set_hwirq_and_chip(d, virq, hwirq,
 705                                                     &gic_level_irq_controller,
 706                                                     NULL);
 707         } else {
 708                 switch (GIC_HWIRQ_TO_LOCAL(hwirq)) {
 709                 case GIC_LOCAL_INT_TIMER:
 710                 case GIC_LOCAL_INT_PERFCTR:
 711                 case GIC_LOCAL_INT_FDC:
 712                         /*
 713                          * HACK: These are all really percpu interrupts, but
 714                          * the rest of the MIPS kernel code does not use the
 715                          * percpu IRQ API for them.
 716                          */
 717                         chip = &gic_all_vpes_local_irq_controller;
 718                         irq_set_handler(virq, handle_percpu_irq);
 719                         break;
 720
 721                 default:
 722                         chip = &gic_local_irq_controller;
 723                         irq_set_handler(virq, handle_percpu_devid_irq);
 724                         irq_set_percpu_devid(virq);
 725                         break;
 726                 }
 727
 728                 err = irq_domain_set_hwirq_and_chip(d, virq, hwirq,
 729                                                     chip, NULL);
 730         }
 731
 732         return err;
 733 }
 734
 735 static int gic_irq_domain_alloc(struct irq_domain *d, unsigned int virq,
 736                                 unsigned int nr_irqs, void *arg)
 737 {
 738         struct gic_irq_spec *spec = arg;
 739         irq_hw_number_t hwirq, base_hwirq;
 740         int cpu, ret, i;
 741
 742         if (spec->type == GIC_DEVICE) {
 743                 /* verify that shared irqs don't conflict with an IPI irq */
 744                 if ((spec->hwirq >= GIC_SHARED_HWIRQ_BASE) &&
 745                     test_bit(GIC_HWIRQ_TO_SHARED(spec->hwirq), ipi_resrv))
 746                         return -EBUSY;
 747
 748                 return gic_setup_dev_chip(d, virq, spec->hwirq);
 749         } else {
 750                 base_hwirq = find_first_bit(ipi_available, gic_shared_intrs);
 751                 if (base_hwirq == gic_shared_intrs) {
 752                         return -ENOMEM;
 753                 }
 754
 755                 /* check that we have enough space */
 756                 for (i = base_hwirq; i < nr_irqs; i++) {
 757                         if (!test_bit(i, ipi_available))
 758                                 return -EBUSY;
 759                 }
 760                 bitmap_clear(ipi_available, base_hwirq, nr_irqs);
 761
 762                 /* map the hwirq for each cpu consecutively */
 763                 i = 0;
 764                 for_each_cpu(cpu, spec->ipimask) {
 765                         hwirq = GIC_SHARED_TO_HWIRQ(base_hwirq + i);
 766
 767                         ret = irq_domain_set_hwirq_and_chip(d, virq + i, hwirq,
 768                                                             &gic_level_irq_controller,
 769                                                             NULL);
 770                         if (ret)
 771                                 goto error;
 772
 773                         irq_set_handler(virq + i, handle_level_irq);
 774
 775                         ret = gic_shared_irq_domain_map(d, virq + i, hwirq, cpu);
 776                         if (ret)
 777                                 goto error;
 778
 779                         i++;
 780                 }
 781
 782                 /*
 783                  * tell the parent about the base hwirq we allocated so it can
 784                  * set its own domain data
 785                  */
 786                 spec->hwirq = base_hwirq;
 787         }
 788
 789         return 0;
 790 error:
 791         bitmap_set(ipi_available, base_hwirq, nr_irqs);
 792         return ret;
 793 }
 794
 795 void gic_irq_domain_free(struct irq_domain *d, unsigned int virq,
 796                          unsigned int nr_irqs)
 797 {
 798         irq_hw_number_t base_hwirq;
 799         struct irq_data *data;
 800
 801         data = irq_get_irq_data(virq);
 802         if (!data)
 803                 return;
 804
 805         base_hwirq = GIC_HWIRQ_TO_SHARED(irqd_to_hwirq(data));
 806         bitmap_set(ipi_available, base_hwirq, nr_irqs);
 807 }
 808
 809 int gic_irq_domain_match(struct irq_domain *d, struct device_node *node,
 810                          enum irq_domain_bus_token bus_token)
 811 {
 812         /* this domain should'nt be accessed directly */
 813         return 0;
 814 }
 815
 816 static const struct irq_domain_ops gic_irq_domain_ops = {
 817         .alloc = gic_irq_domain_alloc,
 818         .free = gic_irq_domain_free,
 819         .match = gic_irq_domain_match,
 820 };
 821
 822 static int gic_dev_domain_xlate(struct irq_domain *d, struct device_node *ctrlr,
 823                                 const u32 *intspec, unsigned int intsize,
 824                                 irq_hw_number_t *out_hwirq,
 825                                 unsigned int *out_type)
 826 {
 827         if (intsize != 3)
 828                 return -EINVAL;
 829
 830         if (intspec[0] == GIC_SHARED)
 831                 *out_hwirq = GIC_SHARED_TO_HWIRQ(intspec[1]);
 832         else if (intspec[0] == GIC_LOCAL)
 833                 *out_hwirq = GIC_LOCAL_TO_HWIRQ(intspec[1]);
 834         else
 835                 return -EINVAL;
 836         *out_type = intspec[2] & IRQ_TYPE_SENSE_MASK;
 837
 838         return 0;
 839 }
 840
 841 static int gic_dev_domain_alloc(struct irq_domain *d, unsigned int virq,
 842                                 unsigned int nr_irqs, void *arg)
 843 {
 844         struct irq_fwspec *fwspec = arg;
 845         struct gic_irq_spec spec = {
 846                 .type = GIC_DEVICE,
 847         };
 848         int i, ret;
 849
 850         if (fwspec->param[0] == GIC_SHARED)
 851                 spec.hwirq = GIC_SHARED_TO_HWIRQ(fwspec->param[1]);
 852         else
 853                 spec.hwirq = GIC_LOCAL_TO_HWIRQ(fwspec->param[1]);
 854
 855         ret = irq_domain_alloc_irqs_parent(d, virq, nr_irqs, &spec);
 856         if (ret)
 857                 return ret;
 858
 859         for (i = 0; i < nr_irqs; i++) {
 860                 ret = gic_setup_dev_chip(d, virq + i, spec.hwirq + i);
 861                 if (ret)
 862                         goto error;
 863         }
 864
 865         return 0;
 866
 867 error:
 868         irq_domain_free_irqs_parent(d, virq, nr_irqs);
 869         return ret;
 870 }
 871
 872 void gic_dev_domain_free(struct irq_domain *d, unsigned int virq,
 873                          unsigned int nr_irqs)
 874 {
 875         /* no real allocation is done for dev irqs, so no need to free anything */
 876         return;
 877 }
 878
 879 static void gic_dev_domain_activate(struct irq_domain *domain,
 880                                     struct irq_data *d)
 881 {
 882         if (GIC_HWIRQ_TO_LOCAL(d->hwirq) < GIC_NUM_LOCAL_INTRS)
 883                 gic_local_irq_domain_map(domain, d->irq, d->hwirq);
 884         else
 885                 gic_shared_irq_domain_map(domain, d->irq, d->hwirq, 0);
 886 }
 887
 888 static struct irq_domain_ops gic_dev_domain_ops = {
 889         .xlate = gic_dev_domain_xlate,
 890         .alloc = gic_dev_domain_alloc,
 891         .free = gic_dev_domain_free,
 892         .activate = gic_dev_domain_activate,
 893 };
 894
 895 static int gic_ipi_domain_xlate(struct irq_domain *d, struct device_node *ctrlr,
 896                                 const u32 *intspec, unsigned int intsize,
 897                                 irq_hw_number_t *out_hwirq,
 898                                 unsigned int *out_type)
 899 {
 900         /*
 901          * There's nothing to translate here. hwirq is dynamically allocated and
 902          * the irq type is always edge triggered.
 903          * */
 904         *out_hwirq = 0;
 905         *out_type = IRQ_TYPE_EDGE_RISING;
 906
 907         return 0;
 908 }
 909
 910 static int gic_ipi_domain_alloc(struct irq_domain *d, unsigned int virq,
 911                                 unsigned int nr_irqs, void *arg)
 912 {
 913         struct cpumask *ipimask = arg;
 914         struct gic_irq_spec spec = {
 915                 .type = GIC_IPI,
 916                 .ipimask = ipimask
 917         };
 918         int ret, i;
 919
 920         ret = irq_domain_alloc_irqs_parent(d, virq, nr_irqs, &spec);
 921         if (ret)
 922                 return ret;
 923
 924         /* the parent should have set spec.hwirq to the base_hwirq it allocated */
 925         for (i = 0; i < nr_irqs; i++) {
 926                 ret = irq_domain_set_hwirq_and_chip(d, virq + i,
 927                                                     GIC_SHARED_TO_HWIRQ(spec.hwirq + i),
 928                                                     &gic_edge_irq_controller,
 929                                                     NULL);
 930                 if (ret)
 931                         goto error;
 932
 933                 ret = irq_set_irq_type(virq + i, IRQ_TYPE_EDGE_RISING);
 934                 if (ret)
 935                         goto error;
 936         }
 937
 938         return 0;
 939 error:
 940         irq_domain_free_irqs_parent(d, virq, nr_irqs);
 941         return ret;
 942 }
 943
 944 void gic_ipi_domain_free(struct irq_domain *d, unsigned int virq,
 945                          unsigned int nr_irqs)
 946 {
 947         irq_domain_free_irqs_parent(d, virq, nr_irqs);
 948 }
 949
 950 int gic_ipi_domain_match(struct irq_domain *d, struct device_node *node,
 951                          enum irq_domain_bus_token bus_token)
 952 {
 953         bool is_ipi;
 954
 955         switch (bus_token) {
 956         case DOMAIN_BUS_IPI:
 957                 is_ipi = d->bus_token == bus_token;
 958                 return (!node || to_of_node(d->fwnode) == node) && is_ipi;
 959                 break;
 960         default:
 961                 return 0;
 962         }
 963 }
 964
 965 static struct irq_domain_ops gic_ipi_domain_ops = {
 966         .xlate = gic_ipi_domain_xlate,
 967         .alloc = gic_ipi_domain_alloc,
 968         .free = gic_ipi_domain_free,
 969         .match = gic_ipi_domain_match,
 970 };
 971
 972 static void __init __gic_init(unsigned long gic_base_addr,
 973                               unsigned long gic_addrspace_size,
 974                               unsigned int cpu_vec, unsigned int irqbase,
 975                               struct device_node *node)
 976 {
 977         unsigned int gicconfig, cpu;
 978         unsigned int v[2];
 979
 980         __gic_base_addr = gic_base_addr;
 981
 982         gic_base = ioremap_nocache(gic_base_addr, gic_addrspace_size);
 983
 984         gicconfig = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
 985         gic_shared_intrs = (gicconfig & GIC_SH_CONFIG_NUMINTRS_MSK) >>
 986                    GIC_SH_CONFIG_NUMINTRS_SHF;
 987         gic_shared_intrs = ((gic_shared_intrs + 1) * 8);
 988
 989         gic_vpes = (gicconfig & GIC_SH_CONFIG_NUMVPES_MSK) >>
 990                   GIC_SH_CONFIG_NUMVPES_SHF;
 991         gic_vpes = gic_vpes + 1;
 992
 993         if (cpu_has_veic) {
 994                 /* Set EIC mode for all VPEs */
 995                 for_each_present_cpu(cpu) {
 996                         gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
 997                                   mips_cm_vp_id(cpu));
 998                         gic_write(GIC_REG(VPE_OTHER, GIC_VPE_CTL),
 999                                   GIC_VPE_CTL_EIC_MODE_MSK);
1000                 }
1001
1002                 /* Always use vector 1 in EIC mode */
1003                 gic_cpu_pin = 0;
1004                 timer_cpu_pin = gic_cpu_pin;
1005                 set_vi_handler(gic_cpu_pin + GIC_PIN_TO_VEC_OFFSET,
1006                                __gic_irq_dispatch);
1007         } else {
1008                 gic_cpu_pin = cpu_vec - GIC_CPU_PIN_OFFSET;
1009                 irq_set_chained_handler(MIPS_CPU_IRQ_BASE + cpu_vec,
1010                                         gic_irq_dispatch);
1011                 /*
1012                  * With the CMP implementation of SMP (deprecated), other CPUs
1013                  * are started by the bootloader and put into a timer based
1014                  * waiting poll loop. We must not re-route those CPU's local
1015                  * timer interrupts as the wait instruction will never finish,
1016                  * so just handle whatever CPU interrupt it is routed to by
1017                  * default.
1018                  *
1019                  * This workaround should be removed when CMP support is
1020                  * dropped.
1021                  */
1022                 if (IS_ENABLED(CONFIG_MIPS_CMP) &&
1023                     gic_local_irq_is_routable(GIC_LOCAL_INT_TIMER)) {
1024                         timer_cpu_pin = gic_read32(GIC_REG(VPE_LOCAL,
1025                                                          GIC_VPE_TIMER_MAP)) &
1026                                         GIC_MAP_MSK;
1027                         irq_set_chained_handler(MIPS_CPU_IRQ_BASE +
1028                                                 GIC_CPU_PIN_OFFSET +
1029                                                 timer_cpu_pin,
1030                                                 gic_irq_dispatch);
1031                 } else {
1032                         timer_cpu_pin = gic_cpu_pin;
1033                 }
1034         }
1035
1036         gic_irq_domain = irq_domain_add_simple(node, GIC_NUM_LOCAL_INTRS +
1037                                                gic_shared_intrs, irqbase,
1038                                                &gic_irq_domain_ops, NULL);
1039         if (!gic_irq_domain)
1040                 panic("Failed to add GIC IRQ domain");
1041         gic_irq_domain->name = "mips-gic-irq";
1042
1043         gic_dev_domain = irq_domain_add_hierarchy(gic_irq_domain, 0,
1044                                                   GIC_NUM_LOCAL_INTRS + gic_shared_intrs,
1045                                                   node, &gic_dev_domain_ops, NULL);
1046         if (!gic_dev_domain)
1047                 panic("Failed to add GIC DEV domain");
1048         gic_dev_domain->name = "mips-gic-dev";
1049
1050         gic_ipi_domain = irq_domain_add_hierarchy(gic_irq_domain,
1051                                                   IRQ_DOMAIN_FLAG_IPI_PER_CPU,
1052                                                   GIC_NUM_LOCAL_INTRS + gic_shared_intrs,
1053                                                   node, &gic_ipi_domain_ops, NULL);
1054         if (!gic_ipi_domain)
1055                 panic("Failed to add GIC IPI domain");
1056
1057         gic_ipi_domain->name = "mips-gic-ipi";
1058         gic_ipi_domain->bus_token = DOMAIN_BUS_IPI;
1059
1060         if (node &&
1061             !of_property_read_u32_array(node, "mti,reserved-ipi-vectors", v, 2)) {
1062                 bitmap_set(ipi_resrv, v[0], v[1]);
1063         } else {
1064                 /* Make the last 2 * gic_vpes available for IPIs */
1065                 bitmap_set(ipi_resrv,
1066                            gic_shared_intrs - 2 * gic_vpes,
1067                            2 * gic_vpes);
1068         }
1069
1070         bitmap_copy(ipi_available, ipi_resrv, GIC_MAX_INTRS);
1071         gic_basic_init();
1072 }
1073
1074 void __init gic_init(unsigned long gic_base_addr,
1075                      unsigned long gic_addrspace_size,
1076                      unsigned int cpu_vec, unsigned int irqbase)
1077 {
1078         __gic_init(gic_base_addr, gic_addrspace_size, cpu_vec, irqbase, NULL);
1079 }
1080
1081 static int __init gic_of_init(struct device_node *node,
1082                               struct device_node *parent)
1083 {
1084         struct resource res;
1085         unsigned int cpu_vec, i = 0, reserved = 0;
1086         phys_addr_t gic_base;
1087         size_t gic_len;
1088
1089         /* Find the first available CPU vector. */
1090         while (!of_property_read_u32_index(node, "mti,reserved-cpu-vectors",
1091                                            i++, &cpu_vec))
1092                 reserved |= BIT(cpu_vec);
1093         for (cpu_vec = 2; cpu_vec < 8; cpu_vec++) {
1094                 if (!(reserved & BIT(cpu_vec)))
1095                         break;
1096         }
1097         if (cpu_vec == 8) {
1098                 pr_err("No CPU vectors available for GIC\n");
1099                 return -ENODEV;
1100         }
1101
1102         if (of_address_to_resource(node, 0, &res)) {
1103                 /*
1104                  * Probe the CM for the GIC base address if not specified
1105                  * in the device-tree.
1106                  */
1107                 if (mips_cm_present()) {
1108                         gic_base = read_gcr_gic_base() &
1109                                 ~CM_GCR_GIC_BASE_GICEN_MSK;
1110                         gic_len = 0x20000;
1111                 } else {
1112                         pr_err("Failed to get GIC memory range\n");
1113                         return -ENODEV;
1114                 }
1115         } else {
1116                 gic_base = res.start;
1117                 gic_len = resource_size(&res);
1118         }
1119
1120         if (mips_cm_present()) {
1121                 write_gcr_gic_base(gic_base | CM_GCR_GIC_BASE_GICEN_MSK);
1122                 /* Ensure GIC region is enabled before trying to access it */
1123                 __sync();
1124         }
1125         gic_present = true;
1126
1127         __gic_init(gic_base, gic_len, cpu_vec, 0, node);
1128
1129         return 0;
1130 }
1131 IRQCHIP_DECLARE(mips_gic, "mti,gic", gic_of_init);