drivers/irqchip/irq-gic-v3-its.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
   4  * Author: Marc Zyngier <marc.zyngier@arm.com>
   5  */
   6
   7 #include <linux/acpi.h>
   8 #include <linux/acpi_iort.h>
   9 #include <linux/bitfield.h>
  10 #include <linux/bitmap.h>
  11 #include <linux/cpu.h>
  12 #include <linux/crash_dump.h>
  13 #include <linux/delay.h>
  14 #include <linux/dma-iommu.h>
  15 #include <linux/efi.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/iopoll.h>
  18 #include <linux/irqdomain.h>
  19 #include <linux/list.h>
  20 #include <linux/log2.h>
  21 #include <linux/memblock.h>
  22 #include <linux/mm.h>
  23 #include <linux/msi.h>
  24 #include <linux/of.h>
  25 #include <linux/of_address.h>
  26 #include <linux/of_irq.h>
  27 #include <linux/of_pci.h>
  28 #include <linux/of_platform.h>
  29 #include <linux/percpu.h>
  30 #include <linux/slab.h>
  31 #include <linux/syscore_ops.h>
  32
  33 #include <linux/irqchip.h>
  34 #include <linux/irqchip/arm-gic-v3.h>
  35 #include <linux/irqchip/arm-gic-v4.h>
  36
  37 #include <asm/cputype.h>
  38 #include <asm/exception.h>
  39
  40 #include "irq-gic-common.h"
  41
  42 #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING           (1ULL << 0)
  43 #define ITS_FLAGS_WORKAROUND_CAVIUM_22375       (1ULL << 1)
  44 #define ITS_FLAGS_WORKAROUND_CAVIUM_23144       (1ULL << 2)
  45
  46 #define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING     (1 << 0)
  47 #define RDIST_FLAGS_RD_TABLES_PREALLOCATED      (1 << 1)
  48
  49 static u32 lpi_id_bits;
  50
  51 /*
  52  * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to
  53  * deal with (one configuration byte per interrupt). PENDBASE has to
  54  * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
  55  */
  56 #define LPI_NRBITS              lpi_id_bits
  57 #define LPI_PROPBASE_SZ         ALIGN(BIT(LPI_NRBITS), SZ_64K)
  58 #define LPI_PENDBASE_SZ         ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K)
  59
  60 #define LPI_PROP_DEFAULT_PRIO   GICD_INT_DEF_PRI
  61
  62 /*
  63  * Collection structure - just an ID, and a redistributor address to
  64  * ping. We use one per CPU as a bag of interrupts assigned to this
  65  * CPU.
  66  */
  67 struct its_collection {
  68         u64                     target_address;
  69         u16                     col_id;
  70 };
  71
  72 /*
  73  * The ITS_BASER structure - contains memory information, cached
  74  * value of BASER register configuration and ITS page size.
  75  */
  76 struct its_baser {
  77         void            *base;
  78         u64             val;
  79         u32             order;
  80         u32             psz;
  81 };
  82
  83 struct its_device;
  84
  85 /*
  86  * The ITS structure - contains most of the infrastructure, with the
  87  * top-level MSI domain, the command queue, the collections, and the
  88  * list of devices writing to it.
  89  *
  90  * dev_alloc_lock has to be taken for device allocations, while the
  91  * spinlock must be taken to parse data structures such as the device
  92  * list.
  93  */
  94 struct its_node {
  95         raw_spinlock_t          lock;
  96         struct mutex            dev_alloc_lock;
  97         struct list_head        entry;
  98         void __iomem            *base;
  99         void __iomem            *sgir_base;
 100         phys_addr_t             phys_base;
 101         struct its_cmd_block    *cmd_base;
 102         struct its_cmd_block    *cmd_write;
 103         struct its_baser        tables[GITS_BASER_NR_REGS];
 104         struct its_collection   *collections;
 105         struct fwnode_handle    *fwnode_handle;
 106         u64                     (*get_msi_base)(struct its_device *its_dev);
 107         u64                     typer;
 108         u64                     cbaser_save;
 109         u32                     ctlr_save;
 110         u32                     mpidr;
 111         struct list_head        its_device_list;
 112         u64                     flags;
 113         unsigned long           list_nr;
 114         int                     numa_node;
 115         unsigned int            msi_domain_flags;
 116         u32                     pre_its_base; /* for Socionext Synquacer */
 117         int                     vlpi_redist_offset;
 118 };
 119
 120 #define is_v4(its)              (!!((its)->typer & GITS_TYPER_VLPIS))
 121 #define is_v4_1(its)            (!!((its)->typer & GITS_TYPER_VMAPP))
 122 #define device_ids(its)         (FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1)
 123
 124 #define ITS_ITT_ALIGN           SZ_256
 125
 126 /* The maximum number of VPEID bits supported by VLPI commands */
 127 #define ITS_MAX_VPEID_BITS                                              \
 128         ({                                                              \
 129                 int nvpeid = 16;                                        \
 130                 if (gic_rdists->has_rvpeid &&                           \
 131                     gic_rdists->gicd_typer2 & GICD_TYPER2_VIL)          \
 132                         nvpeid = 1 + (gic_rdists->gicd_typer2 &         \
 133                                       GICD_TYPER2_VID);                 \
 134                                                                         \
 135                 nvpeid;                                                 \
 136         })
 137 #define ITS_MAX_VPEID           (1 << (ITS_MAX_VPEID_BITS))
 138
 139 /* Convert page order to size in bytes */
 140 #define PAGE_ORDER_TO_SIZE(o)   (PAGE_SIZE << (o))
 141
 142 struct event_lpi_map {
 143         unsigned long           *lpi_map;
 144         u16                     *col_map;
 145         irq_hw_number_t         lpi_base;
 146         int                     nr_lpis;
 147         raw_spinlock_t          vlpi_lock;
 148         struct its_vm           *vm;
 149         struct its_vlpi_map     *vlpi_maps;
 150         int                     nr_vlpis;
 151 };
 152
 153 /*
 154  * The ITS view of a device - belongs to an ITS, owns an interrupt
 155  * translation table, and a list of interrupts.  If it some of its
 156  * LPIs are injected into a guest (GICv4), the event_map.vm field
 157  * indicates which one.
 158  */
 159 struct its_device {
 160         struct list_head        entry;
 161         struct its_node         *its;
 162         struct event_lpi_map    event_map;
 163         void                    *itt;
 164         u32                     nr_ites;
 165         u32                     device_id;
 166         bool                    shared;
 167 };
 168
 169 static struct {
 170         raw_spinlock_t          lock;
 171         struct its_device       *dev;
 172         struct its_vpe          **vpes;
 173         int                     next_victim;
 174 } vpe_proxy;
 175
 176 struct cpu_lpi_count {
 177         atomic_t        managed;
 178         atomic_t        unmanaged;
 179 };
 180
 181 static DEFINE_PER_CPU(struct cpu_lpi_count, cpu_lpi_count);
 182
 183 static LIST_HEAD(its_nodes);
 184 static DEFINE_RAW_SPINLOCK(its_lock);
 185 static struct rdists *gic_rdists;
 186 static struct irq_domain *its_parent;
 187
 188 static unsigned long its_list_map;
 189 static u16 vmovp_seq_num;
 190 static DEFINE_RAW_SPINLOCK(vmovp_lock);
 191
 192 static DEFINE_IDA(its_vpeid_ida);
 193
 194 #define gic_data_rdist()                (raw_cpu_ptr(gic_rdists->rdist))
 195 #define gic_data_rdist_cpu(cpu)         (per_cpu_ptr(gic_rdists->rdist, cpu))
 196 #define gic_data_rdist_rd_base()        (gic_data_rdist()->rd_base)
 197 #define gic_data_rdist_vlpi_base()      (gic_data_rdist_rd_base() + SZ_128K)
 198
 199 /*
 200  * Skip ITSs that have no vLPIs mapped, unless we're on GICv4.1, as we
 201  * always have vSGIs mapped.
 202  */
 203 static bool require_its_list_vmovp(struct its_vm *vm, struct its_node *its)
 204 {
 205         return (gic_rdists->has_rvpeid || vm->vlpi_count[its->list_nr]);
 206 }
 207
 208 static u16 get_its_list(struct its_vm *vm)
 209 {
 210         struct its_node *its;
 211         unsigned long its_list = 0;
 212
 213         list_for_each_entry(its, &its_nodes, entry) {
 214                 if (!is_v4(its))
 215                         continue;
 216
 217                 if (require_its_list_vmovp(vm, its))
 218                         __set_bit(its->list_nr, &its_list);
 219         }
 220
 221         return (u16)its_list;
 222 }
 223
 224 static inline u32 its_get_event_id(struct irq_data *d)
 225 {
 226         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 227         return d->hwirq - its_dev->event_map.lpi_base;
 228 }
 229
 230 static struct its_collection *dev_event_to_col(struct its_device *its_dev,
 231                                                u32 event)
 232 {
 233         struct its_node *its = its_dev->its;
 234
 235         return its->collections + its_dev->event_map.col_map[event];
 236 }
 237
 238 static struct its_vlpi_map *dev_event_to_vlpi_map(struct its_device *its_dev,
 239                                                u32 event)
 240 {
 241         if (WARN_ON_ONCE(event >= its_dev->event_map.nr_lpis))
 242                 return NULL;
 243
 244         return &its_dev->event_map.vlpi_maps[event];
 245 }
 246
 247 static struct its_vlpi_map *get_vlpi_map(struct irq_data *d)
 248 {
 249         if (irqd_is_forwarded_to_vcpu(d)) {
 250                 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 251                 u32 event = its_get_event_id(d);
 252
 253                 return dev_event_to_vlpi_map(its_dev, event);
 254         }
 255
 256         return NULL;
 257 }
 258
 259 static int vpe_to_cpuid_lock(struct its_vpe *vpe, unsigned long *flags)
 260 {
 261         raw_spin_lock_irqsave(&vpe->vpe_lock, *flags);
 262         return vpe->col_idx;
 263 }
 264
 265 static void vpe_to_cpuid_unlock(struct its_vpe *vpe, unsigned long flags)
 266 {
 267         raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
 268 }
 269
 270 static struct irq_chip its_vpe_irq_chip;
 271
 272 static int irq_to_cpuid_lock(struct irq_data *d, unsigned long *flags)
 273 {
 274         struct its_vpe *vpe = NULL;
 275         int cpu;
 276
 277         if (d->chip == &its_vpe_irq_chip) {
 278                 vpe = irq_data_get_irq_chip_data(d);
 279         } else {
 280                 struct its_vlpi_map *map = get_vlpi_map(d);
 281                 if (map)
 282                         vpe = map->vpe;
 283         }
 284
 285         if (vpe) {
 286                 cpu = vpe_to_cpuid_lock(vpe, flags);
 287         } else {
 288                 /* Physical LPIs are already locked via the irq_desc lock */
 289                 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 290                 cpu = its_dev->event_map.col_map[its_get_event_id(d)];
 291                 /* Keep GCC quiet... */
 292                 *flags = 0;
 293         }
 294
 295         return cpu;
 296 }
 297
 298 static void irq_to_cpuid_unlock(struct irq_data *d, unsigned long flags)
 299 {
 300         struct its_vpe *vpe = NULL;
 301
 302         if (d->chip == &its_vpe_irq_chip) {
 303                 vpe = irq_data_get_irq_chip_data(d);
 304         } else {
 305                 struct its_vlpi_map *map = get_vlpi_map(d);
 306                 if (map)
 307                         vpe = map->vpe;
 308         }
 309
 310         if (vpe)
 311                 vpe_to_cpuid_unlock(vpe, flags);
 312 }
 313
 314 static struct its_collection *valid_col(struct its_collection *col)
 315 {
 316         if (WARN_ON_ONCE(col->target_address & GENMASK_ULL(15, 0)))
 317                 return NULL;
 318
 319         return col;
 320 }
 321
 322 static struct its_vpe *valid_vpe(struct its_node *its, struct its_vpe *vpe)
 323 {
 324         if (valid_col(its->collections + vpe->col_idx))
 325                 return vpe;
 326
 327         return NULL;
 328 }
 329
 330 /*
 331  * ITS command descriptors - parameters to be encoded in a command
 332  * block.
 333  */
 334 struct its_cmd_desc {
 335         union {
 336                 struct {
 337                         struct its_device *dev;
 338                         u32 event_id;
 339                 } its_inv_cmd;
 340
 341                 struct {
 342                         struct its_device *dev;
 343                         u32 event_id;
 344                 } its_clear_cmd;
 345
 346                 struct {
 347                         struct its_device *dev;
 348                         u32 event_id;
 349                 } its_int_cmd;
 350
 351                 struct {
 352                         struct its_device *dev;
 353                         int valid;
 354                 } its_mapd_cmd;
 355
 356                 struct {
 357                         struct its_collection *col;
 358                         int valid;
 359                 } its_mapc_cmd;
 360
 361                 struct {
 362                         struct its_device *dev;
 363                         u32 phys_id;
 364                         u32 event_id;
 365                 } its_mapti_cmd;
 366
 367                 struct {
 368                         struct its_device *dev;
 369                         struct its_collection *col;
 370                         u32 event_id;
 371                 } its_movi_cmd;
 372
 373                 struct {
 374                         struct its_device *dev;
 375                         u32 event_id;
 376                 } its_discard_cmd;
 377
 378                 struct {
 379                         struct its_collection *col;
 380                 } its_invall_cmd;
 381
 382                 struct {
 383                         struct its_vpe *vpe;
 384                 } its_vinvall_cmd;
 385
 386                 struct {
 387                         struct its_vpe *vpe;
 388                         struct its_collection *col;
 389                         bool valid;
 390                 } its_vmapp_cmd;
 391
 392                 struct {
 393                         struct its_vpe *vpe;
 394                         struct its_device *dev;
 395                         u32 virt_id;
 396                         u32 event_id;
 397                         bool db_enabled;
 398                 } its_vmapti_cmd;
 399
 400                 struct {
 401                         struct its_vpe *vpe;
 402                         struct its_device *dev;
 403                         u32 event_id;
 404                         bool db_enabled;
 405                 } its_vmovi_cmd;
 406
 407                 struct {
 408                         struct its_vpe *vpe;
 409                         struct its_collection *col;
 410                         u16 seq_num;
 411                         u16 its_list;
 412                 } its_vmovp_cmd;
 413
 414                 struct {
 415                         struct its_vpe *vpe;
 416                 } its_invdb_cmd;
 417
 418                 struct {
 419                         struct its_vpe *vpe;
 420                         u8 sgi;
 421                         u8 priority;
 422                         bool enable;
 423                         bool group;
 424                         bool clear;
 425                 } its_vsgi_cmd;
 426         };
 427 };
 428
 429 /*
 430  * The ITS command block, which is what the ITS actually parses.
 431  */
 432 struct its_cmd_block {
 433         union {
 434                 u64     raw_cmd[4];
 435                 __le64  raw_cmd_le[4];
 436         };
 437 };
 438
 439 #define ITS_CMD_QUEUE_SZ                SZ_64K
 440 #define ITS_CMD_QUEUE_NR_ENTRIES        (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block))
 441
 442 typedef struct its_collection *(*its_cmd_builder_t)(struct its_node *,
 443                                                     struct its_cmd_block *,
 444                                                     struct its_cmd_desc *);
 445
 446 typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_node *,
 447                                               struct its_cmd_block *,
 448                                               struct its_cmd_desc *);
 449
 450 static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l)
 451 {
 452         u64 mask = GENMASK_ULL(h, l);
 453         *raw_cmd &= ~mask;
 454         *raw_cmd |= (val << l) & mask;
 455 }
 456
 457 static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr)
 458 {
 459         its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0);
 460 }
 461
 462 static void its_encode_devid(struct its_cmd_block *cmd, u32 devid)
 463 {
 464         its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32);
 465 }
 466
 467 static void its_encode_event_id(struct its_cmd_block *cmd, u32 id)
 468 {
 469         its_mask_encode(&cmd->raw_cmd[1], id, 31, 0);
 470 }
 471
 472 static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id)
 473 {
 474         its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32);
 475 }
 476
 477 static void its_encode_size(struct its_cmd_block *cmd, u8 size)
 478 {
 479         its_mask_encode(&cmd->raw_cmd[1], size, 4, 0);
 480 }
 481
 482 static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr)
 483 {
 484         its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8);
 485 }
 486
 487 static void its_encode_valid(struct its_cmd_block *cmd, int valid)
 488 {
 489         its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63);
 490 }
 491
 492 static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr)
 493 {
 494         its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16);
 495 }
 496
 497 static void its_encode_collection(struct its_cmd_block *cmd, u16 col)
 498 {
 499         its_mask_encode(&cmd->raw_cmd[2], col, 15, 0);
 500 }
 501
 502 static void its_encode_vpeid(struct its_cmd_block *cmd, u16 vpeid)
 503 {
 504         its_mask_encode(&cmd->raw_cmd[1], vpeid, 47, 32);
 505 }
 506
 507 static void its_encode_virt_id(struct its_cmd_block *cmd, u32 virt_id)
 508 {
 509         its_mask_encode(&cmd->raw_cmd[2], virt_id, 31, 0);
 510 }
 511
 512 static void its_encode_db_phys_id(struct its_cmd_block *cmd, u32 db_phys_id)
 513 {
 514         its_mask_encode(&cmd->raw_cmd[2], db_phys_id, 63, 32);
 515 }
 516
 517 static void its_encode_db_valid(struct its_cmd_block *cmd, bool db_valid)
 518 {
 519         its_mask_encode(&cmd->raw_cmd[2], db_valid, 0, 0);
 520 }
 521
 522 static void its_encode_seq_num(struct its_cmd_block *cmd, u16 seq_num)
 523 {
 524         its_mask_encode(&cmd->raw_cmd[0], seq_num, 47, 32);
 525 }
 526
 527 static void its_encode_its_list(struct its_cmd_block *cmd, u16 its_list)
 528 {
 529         its_mask_encode(&cmd->raw_cmd[1], its_list, 15, 0);
 530 }
 531
 532 static void its_encode_vpt_addr(struct its_cmd_block *cmd, u64 vpt_pa)
 533 {
 534         its_mask_encode(&cmd->raw_cmd[3], vpt_pa >> 16, 51, 16);
 535 }
 536
 537 static void its_encode_vpt_size(struct its_cmd_block *cmd, u8 vpt_size)
 538 {
 539         its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0);
 540 }
 541
 542 static void its_encode_vconf_addr(struct its_cmd_block *cmd, u64 vconf_pa)
 543 {
 544         its_mask_encode(&cmd->raw_cmd[0], vconf_pa >> 16, 51, 16);
 545 }
 546
 547 static void its_encode_alloc(struct its_cmd_block *cmd, bool alloc)
 548 {
 549         its_mask_encode(&cmd->raw_cmd[0], alloc, 8, 8);
 550 }
 551
 552 static void its_encode_ptz(struct its_cmd_block *cmd, bool ptz)
 553 {
 554         its_mask_encode(&cmd->raw_cmd[0], ptz, 9, 9);
 555 }
 556
 557 static void its_encode_vmapp_default_db(struct its_cmd_block *cmd,
 558                                         u32 vpe_db_lpi)
 559 {
 560         its_mask_encode(&cmd->raw_cmd[1], vpe_db_lpi, 31, 0);
 561 }
 562
 563 static void its_encode_vmovp_default_db(struct its_cmd_block *cmd,
 564                                         u32 vpe_db_lpi)
 565 {
 566         its_mask_encode(&cmd->raw_cmd[3], vpe_db_lpi, 31, 0);
 567 }
 568
 569 static void its_encode_db(struct its_cmd_block *cmd, bool db)
 570 {
 571         its_mask_encode(&cmd->raw_cmd[2], db, 63, 63);
 572 }
 573
 574 static void its_encode_sgi_intid(struct its_cmd_block *cmd, u8 sgi)
 575 {
 576         its_mask_encode(&cmd->raw_cmd[0], sgi, 35, 32);
 577 }
 578
 579 static void its_encode_sgi_priority(struct its_cmd_block *cmd, u8 prio)
 580 {
 581         its_mask_encode(&cmd->raw_cmd[0], prio >> 4, 23, 20);
 582 }
 583
 584 static void its_encode_sgi_group(struct its_cmd_block *cmd, bool grp)
 585 {
 586         its_mask_encode(&cmd->raw_cmd[0], grp, 10, 10);
 587 }
 588
 589 static void its_encode_sgi_clear(struct its_cmd_block *cmd, bool clr)
 590 {
 591         its_mask_encode(&cmd->raw_cmd[0], clr, 9, 9);
 592 }
 593
 594 static void its_encode_sgi_enable(struct its_cmd_block *cmd, bool en)
 595 {
 596         its_mask_encode(&cmd->raw_cmd[0], en, 8, 8);
 597 }
 598
 599 static inline void its_fixup_cmd(struct its_cmd_block *cmd)
 600 {
 601         /* Let's fixup BE commands */
 602         cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]);
 603         cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]);
 604         cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]);
 605         cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]);
 606 }
 607
 608 static struct its_collection *its_build_mapd_cmd(struct its_node *its,
 609                                                  struct its_cmd_block *cmd,
 610                                                  struct its_cmd_desc *desc)
 611 {
 612         unsigned long itt_addr;
 613         u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites);
 614
 615         itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt);
 616         itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN);
 617
 618         its_encode_cmd(cmd, GITS_CMD_MAPD);
 619         its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id);
 620         its_encode_size(cmd, size - 1);
 621         its_encode_itt(cmd, itt_addr);
 622         its_encode_valid(cmd, desc->its_mapd_cmd.valid);
 623
 624         its_fixup_cmd(cmd);
 625
 626         return NULL;
 627 }
 628
 629 static struct its_collection *its_build_mapc_cmd(struct its_node *its,
 630                                                  struct its_cmd_block *cmd,
 631                                                  struct its_cmd_desc *desc)
 632 {
 633         its_encode_cmd(cmd, GITS_CMD_MAPC);
 634         its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);
 635         its_encode_target(cmd, desc->its_mapc_cmd.col->target_address);
 636         its_encode_valid(cmd, desc->its_mapc_cmd.valid);
 637
 638         its_fixup_cmd(cmd);
 639
 640         return desc->its_mapc_cmd.col;
 641 }
 642
 643 static struct its_collection *its_build_mapti_cmd(struct its_node *its,
 644                                                   struct its_cmd_block *cmd,
 645                                                   struct its_cmd_desc *desc)
 646 {
 647         struct its_collection *col;
 648
 649         col = dev_event_to_col(desc->its_mapti_cmd.dev,
 650                                desc->its_mapti_cmd.event_id);
 651
 652         its_encode_cmd(cmd, GITS_CMD_MAPTI);
 653         its_encode_devid(cmd, desc->its_mapti_cmd.dev->device_id);
 654         its_encode_event_id(cmd, desc->its_mapti_cmd.event_id);
 655         its_encode_phys_id(cmd, desc->its_mapti_cmd.phys_id);
 656         its_encode_collection(cmd, col->col_id);
 657
 658         its_fixup_cmd(cmd);
 659
 660         return valid_col(col);
 661 }
 662
 663 static struct its_collection *its_build_movi_cmd(struct its_node *its,
 664                                                  struct its_cmd_block *cmd,
 665                                                  struct its_cmd_desc *desc)
 666 {
 667         struct its_collection *col;
 668
 669         col = dev_event_to_col(desc->its_movi_cmd.dev,
 670                                desc->its_movi_cmd.event_id);
 671
 672         its_encode_cmd(cmd, GITS_CMD_MOVI);
 673         its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id);
 674         its_encode_event_id(cmd, desc->its_movi_cmd.event_id);
 675         its_encode_collection(cmd, desc->its_movi_cmd.col->col_id);
 676
 677         its_fixup_cmd(cmd);
 678
 679         return valid_col(col);
 680 }
 681
 682 static struct its_collection *its_build_discard_cmd(struct its_node *its,
 683                                                     struct its_cmd_block *cmd,
 684                                                     struct its_cmd_desc *desc)
 685 {
 686         struct its_collection *col;
 687
 688         col = dev_event_to_col(desc->its_discard_cmd.dev,
 689                                desc->its_discard_cmd.event_id);
 690
 691         its_encode_cmd(cmd, GITS_CMD_DISCARD);
 692         its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id);
 693         its_encode_event_id(cmd, desc->its_discard_cmd.event_id);
 694
 695         its_fixup_cmd(cmd);
 696
 697         return valid_col(col);
 698 }
 699
 700 static struct its_collection *its_build_inv_cmd(struct its_node *its,
 701                                                 struct its_cmd_block *cmd,
 702                                                 struct its_cmd_desc *desc)
 703 {
 704         struct its_collection *col;
 705
 706         col = dev_event_to_col(desc->its_inv_cmd.dev,
 707                                desc->its_inv_cmd.event_id);
 708
 709         its_encode_cmd(cmd, GITS_CMD_INV);
 710         its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
 711         its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
 712
 713         its_fixup_cmd(cmd);
 714
 715         return valid_col(col);
 716 }
 717
 718 static struct its_collection *its_build_int_cmd(struct its_node *its,
 719                                                 struct its_cmd_block *cmd,
 720                                                 struct its_cmd_desc *desc)
 721 {
 722         struct its_collection *col;
 723
 724         col = dev_event_to_col(desc->its_int_cmd.dev,
 725                                desc->its_int_cmd.event_id);
 726
 727         its_encode_cmd(cmd, GITS_CMD_INT);
 728         its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
 729         its_encode_event_id(cmd, desc->its_int_cmd.event_id);
 730
 731         its_fixup_cmd(cmd);
 732
 733         return valid_col(col);
 734 }
 735
 736 static struct its_collection *its_build_clear_cmd(struct its_node *its,
 737                                                   struct its_cmd_block *cmd,
 738                                                   struct its_cmd_desc *desc)
 739 {
 740         struct its_collection *col;
 741
 742         col = dev_event_to_col(desc->its_clear_cmd.dev,
 743                                desc->its_clear_cmd.event_id);
 744
 745         its_encode_cmd(cmd, GITS_CMD_CLEAR);
 746         its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
 747         its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
 748
 749         its_fixup_cmd(cmd);
 750
 751         return valid_col(col);
 752 }
 753
 754 static struct its_collection *its_build_invall_cmd(struct its_node *its,
 755                                                    struct its_cmd_block *cmd,
 756                                                    struct its_cmd_desc *desc)
 757 {
 758         its_encode_cmd(cmd, GITS_CMD_INVALL);
 759         its_encode_collection(cmd, desc->its_invall_cmd.col->col_id);
 760
 761         its_fixup_cmd(cmd);
 762
 763         return desc->its_invall_cmd.col;
 764 }
 765
 766 static struct its_vpe *its_build_vinvall_cmd(struct its_node *its,
 767                                              struct its_cmd_block *cmd,
 768                                              struct its_cmd_desc *desc)
 769 {
 770         its_encode_cmd(cmd, GITS_CMD_VINVALL);
 771         its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id);
 772
 773         its_fixup_cmd(cmd);
 774
 775         return valid_vpe(its, desc->its_vinvall_cmd.vpe);
 776 }
 777
 778 static struct its_vpe *its_build_vmapp_cmd(struct its_node *its,
 779                                            struct its_cmd_block *cmd,
 780                                            struct its_cmd_desc *desc)
 781 {
 782         unsigned long vpt_addr, vconf_addr;
 783         u64 target;
 784         bool alloc;
 785
 786         its_encode_cmd(cmd, GITS_CMD_VMAPP);
 787         its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id);
 788         its_encode_valid(cmd, desc->its_vmapp_cmd.valid);
 789
 790         if (!desc->its_vmapp_cmd.valid) {
 791                 if (is_v4_1(its)) {
 792                         alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count);
 793                         its_encode_alloc(cmd, alloc);
 794                 }
 795
 796                 goto out;
 797         }
 798
 799         vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));
 800         target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset;
 801
 802         its_encode_target(cmd, target);
 803         its_encode_vpt_addr(cmd, vpt_addr);
 804         its_encode_vpt_size(cmd, LPI_NRBITS - 1);
 805
 806         if (!is_v4_1(its))
 807                 goto out;
 808
 809         vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page));
 810
 811         alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count);
 812
 813         its_encode_alloc(cmd, alloc);
 814
 815         /* We can only signal PTZ when alloc==1. Why do we have two bits? */
 816         its_encode_ptz(cmd, alloc);
 817         its_encode_vconf_addr(cmd, vconf_addr);
 818         its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi);
 819
 820 out:
 821         its_fixup_cmd(cmd);
 822
 823         return valid_vpe(its, desc->its_vmapp_cmd.vpe);
 824 }
 825
 826 static struct its_vpe *its_build_vmapti_cmd(struct its_node *its,
 827                                             struct its_cmd_block *cmd,
 828                                             struct its_cmd_desc *desc)
 829 {
 830         u32 db;
 831
 832         if (!is_v4_1(its) && desc->its_vmapti_cmd.db_enabled)
 833                 db = desc->its_vmapti_cmd.vpe->vpe_db_lpi;
 834         else
 835                 db = 1023;
 836
 837         its_encode_cmd(cmd, GITS_CMD_VMAPTI);
 838         its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id);
 839         its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id);
 840         its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id);
 841         its_encode_db_phys_id(cmd, db);
 842         its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id);
 843
 844         its_fixup_cmd(cmd);
 845
 846         return valid_vpe(its, desc->its_vmapti_cmd.vpe);
 847 }
 848
 849 static struct its_vpe *its_build_vmovi_cmd(struct its_node *its,
 850                                            struct its_cmd_block *cmd,
 851                                            struct its_cmd_desc *desc)
 852 {
 853         u32 db;
 854
 855         if (!is_v4_1(its) && desc->its_vmovi_cmd.db_enabled)
 856                 db = desc->its_vmovi_cmd.vpe->vpe_db_lpi;
 857         else
 858                 db = 1023;
 859
 860         its_encode_cmd(cmd, GITS_CMD_VMOVI);
 861         its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id);
 862         its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id);
 863         its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id);
 864         its_encode_db_phys_id(cmd, db);
 865         its_encode_db_valid(cmd, true);
 866
 867         its_fixup_cmd(cmd);
 868
 869         return valid_vpe(its, desc->its_vmovi_cmd.vpe);
 870 }
 871
 872 static struct its_vpe *its_build_vmovp_cmd(struct its_node *its,
 873                                            struct its_cmd_block *cmd,
 874                                            struct its_cmd_desc *desc)
 875 {
 876         u64 target;
 877
 878         target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset;
 879         its_encode_cmd(cmd, GITS_CMD_VMOVP);
 880         its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num);
 881         its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list);
 882         its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id);
 883         its_encode_target(cmd, target);
 884
 885         if (is_v4_1(its)) {
 886                 its_encode_db(cmd, true);
 887                 its_encode_vmovp_default_db(cmd, desc->its_vmovp_cmd.vpe->vpe_db_lpi);
 888         }
 889
 890         its_fixup_cmd(cmd);
 891
 892         return valid_vpe(its, desc->its_vmovp_cmd.vpe);
 893 }
 894
 895 static struct its_vpe *its_build_vinv_cmd(struct its_node *its,
 896                                           struct its_cmd_block *cmd,
 897                                           struct its_cmd_desc *desc)
 898 {
 899         struct its_vlpi_map *map;
 900
 901         map = dev_event_to_vlpi_map(desc->its_inv_cmd.dev,
 902                                     desc->its_inv_cmd.event_id);
 903
 904         its_encode_cmd(cmd, GITS_CMD_INV);
 905         its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
 906         its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
 907
 908         its_fixup_cmd(cmd);
 909
 910         return valid_vpe(its, map->vpe);
 911 }
 912
 913 static struct its_vpe *its_build_vint_cmd(struct its_node *its,
 914                                           struct its_cmd_block *cmd,
 915                                           struct its_cmd_desc *desc)
 916 {
 917         struct its_vlpi_map *map;
 918
 919         map = dev_event_to_vlpi_map(desc->its_int_cmd.dev,
 920                                     desc->its_int_cmd.event_id);
 921
 922         its_encode_cmd(cmd, GITS_CMD_INT);
 923         its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
 924         its_encode_event_id(cmd, desc->its_int_cmd.event_id);
 925
 926         its_fixup_cmd(cmd);
 927
 928         return valid_vpe(its, map->vpe);
 929 }
 930
 931 static struct its_vpe *its_build_vclear_cmd(struct its_node *its,
 932                                             struct its_cmd_block *cmd,
 933                                             struct its_cmd_desc *desc)
 934 {
 935         struct its_vlpi_map *map;
 936
 937         map = dev_event_to_vlpi_map(desc->its_clear_cmd.dev,
 938                                     desc->its_clear_cmd.event_id);
 939
 940         its_encode_cmd(cmd, GITS_CMD_CLEAR);
 941         its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
 942         its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
 943
 944         its_fixup_cmd(cmd);
 945
 946         return valid_vpe(its, map->vpe);
 947 }
 948
 949 static struct its_vpe *its_build_invdb_cmd(struct its_node *its,
 950                                            struct its_cmd_block *cmd,
 951                                            struct its_cmd_desc *desc)
 952 {
 953         if (WARN_ON(!is_v4_1(its)))
 954                 return NULL;
 955
 956         its_encode_cmd(cmd, GITS_CMD_INVDB);
 957         its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id);
 958
 959         its_fixup_cmd(cmd);
 960
 961         return valid_vpe(its, desc->its_invdb_cmd.vpe);
 962 }
 963
 964 static struct its_vpe *its_build_vsgi_cmd(struct its_node *its,
 965                                           struct its_cmd_block *cmd,
 966                                           struct its_cmd_desc *desc)
 967 {
 968         if (WARN_ON(!is_v4_1(its)))
 969                 return NULL;
 970
 971         its_encode_cmd(cmd, GITS_CMD_VSGI);
 972         its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id);
 973         its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi);
 974         its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority);
 975         its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group);
 976         its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear);
 977         its_encode_sgi_enable(cmd, desc->its_vsgi_cmd.enable);
 978
 979         its_fixup_cmd(cmd);
 980
 981         return valid_vpe(its, desc->its_vsgi_cmd.vpe);
 982 }
 983
 984 static u64 its_cmd_ptr_to_offset(struct its_node *its,
 985                                  struct its_cmd_block *ptr)
 986 {
 987         return (ptr - its->cmd_base) * sizeof(*ptr);
 988 }
 989
 990 static int its_queue_full(struct its_node *its)
 991 {
 992         int widx;
 993         int ridx;
 994
 995         widx = its->cmd_write - its->cmd_base;
 996         ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block);
 997
 998         /* This is incredibly unlikely to happen, unless the ITS locks up. */
 999         if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx)
1000                 return 1;
1001
1002         return 0;
1003 }
1004
1005 static struct its_cmd_block *its_allocate_entry(struct its_node *its)
1006 {
1007         struct its_cmd_block *cmd;
1008         u32 count = 1000000;    /* 1s! */
1009
1010         while (its_queue_full(its)) {
1011                 count--;
1012                 if (!count) {
1013                         pr_err_ratelimited("ITS queue not draining\n");
1014                         return NULL;
1015                 }
1016                 cpu_relax();
1017                 udelay(1);
1018         }
1019
1020         cmd = its->cmd_write++;
1021
1022         /* Handle queue wrapping */
1023         if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES))
1024                 its->cmd_write = its->cmd_base;
1025
1026         /* Clear command  */
1027         cmd->raw_cmd[0] = 0;
1028         cmd->raw_cmd[1] = 0;
1029         cmd->raw_cmd[2] = 0;
1030         cmd->raw_cmd[3] = 0;
1031
1032         return cmd;
1033 }
1034
1035 static struct its_cmd_block *its_post_commands(struct its_node *its)
1036 {
1037         u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write);
1038
1039         writel_relaxed(wr, its->base + GITS_CWRITER);
1040
1041         return its->cmd_write;
1042 }
1043
1044 static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd)
1045 {
1046         /*
1047          * Make sure the commands written to memory are observable by
1048          * the ITS.
1049          */
1050         if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING)
1051                 gic_flush_dcache_to_poc(cmd, sizeof(*cmd));
1052         else
1053                 dsb(ishst);
1054 }
1055
1056 static int its_wait_for_range_completion(struct its_node *its,
1057                                          u64    prev_idx,
1058                                          struct its_cmd_block *to)
1059 {
1060         u64 rd_idx, to_idx, linear_idx;
1061         u32 count = 1000000;    /* 1s! */
1062
1063         /* Linearize to_idx if the command set has wrapped around */
1064         to_idx = its_cmd_ptr_to_offset(its, to);
1065         if (to_idx < prev_idx)
1066                 to_idx += ITS_CMD_QUEUE_SZ;
1067
1068         linear_idx = prev_idx;
1069
1070         while (1) {
1071                 s64 delta;
1072
1073                 rd_idx = readl_relaxed(its->base + GITS_CREADR);
1074
1075                 /*
1076                  * Compute the read pointer progress, taking the
1077                  * potential wrap-around into account.
1078                  */
1079                 delta = rd_idx - prev_idx;
1080                 if (rd_idx < prev_idx)
1081                         delta += ITS_CMD_QUEUE_SZ;
1082
1083                 linear_idx += delta;
1084                 if (linear_idx >= to_idx)
1085                         break;
1086
1087                 count--;
1088                 if (!count) {
1089                         pr_err_ratelimited("ITS queue timeout (%llu %llu)\n",
1090                                            to_idx, linear_idx);
1091                         return -1;
1092                 }
1093                 prev_idx = rd_idx;
1094                 cpu_relax();
1095                 udelay(1);
1096         }
1097
1098         return 0;
1099 }
1100
1101 /* Warning, macro hell follows */
1102 #define BUILD_SINGLE_CMD_FUNC(name, buildtype, synctype, buildfn)       \
1103 void name(struct its_node *its,                                         \
1104           buildtype builder,                                            \
1105           struct its_cmd_desc *desc)                                    \
1106 {                                                                       \
1107         struct its_cmd_block *cmd, *sync_cmd, *next_cmd;                \
1108         synctype *sync_obj;                                             \
1109         unsigned long flags;                                            \
1110         u64 rd_idx;                                                     \
1111                                                                         \
1112         raw_spin_lock_irqsave(&its->lock, flags);                       \
1113                                                                         \
1114         cmd = its_allocate_entry(its);                                  \
1115         if (!cmd) {             /* We're soooooo screewed... */         \
1116                 raw_spin_unlock_irqrestore(&its->lock, flags);          \
1117                 return;                                                 \
1118         }                                                               \
1119         sync_obj = builder(its, cmd, desc);                             \
1120         its_flush_cmd(its, cmd);                                        \
1121                                                                         \
1122         if (sync_obj) {                                                 \
1123                 sync_cmd = its_allocate_entry(its);                     \
1124                 if (!sync_cmd)                                          \
1125                         goto post;                                      \
1126                                                                         \
1127                 buildfn(its, sync_cmd, sync_obj);                       \
1128                 its_flush_cmd(its, sync_cmd);                           \
1129         }                                                               \
1130                                                                         \
1131 post:                                                                   \
1132         rd_idx = readl_relaxed(its->base + GITS_CREADR);                \
1133         next_cmd = its_post_commands(its);                              \
1134         raw_spin_unlock_irqrestore(&its->lock, flags);                  \
1135                                                                         \
1136         if (its_wait_for_range_completion(its, rd_idx, next_cmd))       \
1137                 pr_err_ratelimited("ITS cmd %ps failed\n", builder);    \
1138 }
1139
1140 static void its_build_sync_cmd(struct its_node *its,
1141                                struct its_cmd_block *sync_cmd,
1142                                struct its_collection *sync_col)
1143 {
1144         its_encode_cmd(sync_cmd, GITS_CMD_SYNC);
1145         its_encode_target(sync_cmd, sync_col->target_address);
1146
1147         its_fixup_cmd(sync_cmd);
1148 }
1149
1150 static BUILD_SINGLE_CMD_FUNC(its_send_single_command, its_cmd_builder_t,
1151                              struct its_collection, its_build_sync_cmd)
1152
1153 static void its_build_vsync_cmd(struct its_node *its,
1154                                 struct its_cmd_block *sync_cmd,
1155                                 struct its_vpe *sync_vpe)
1156 {
1157         its_encode_cmd(sync_cmd, GITS_CMD_VSYNC);
1158         its_encode_vpeid(sync_cmd, sync_vpe->vpe_id);
1159
1160         its_fixup_cmd(sync_cmd);
1161 }
1162
1163 static BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand, its_cmd_vbuilder_t,
1164                              struct its_vpe, its_build_vsync_cmd)
1165
1166 static void its_send_int(struct its_device *dev, u32 event_id)
1167 {
1168         struct its_cmd_desc desc;
1169
1170         desc.its_int_cmd.dev = dev;
1171         desc.its_int_cmd.event_id = event_id;
1172
1173         its_send_single_command(dev->its, its_build_int_cmd, &desc);
1174 }
1175
1176 static void its_send_clear(struct its_device *dev, u32 event_id)
1177 {
1178         struct its_cmd_desc desc;
1179
1180         desc.its_clear_cmd.dev = dev;
1181         desc.its_clear_cmd.event_id = event_id;
1182
1183         its_send_single_command(dev->its, its_build_clear_cmd, &desc);
1184 }
1185
1186 static void its_send_inv(struct its_device *dev, u32 event_id)
1187 {
1188         struct its_cmd_desc desc;
1189
1190         desc.its_inv_cmd.dev = dev;
1191         desc.its_inv_cmd.event_id = event_id;
1192
1193         its_send_single_command(dev->its, its_build_inv_cmd, &desc);
1194 }
1195
1196 static void its_send_mapd(struct its_device *dev, int valid)
1197 {
1198         struct its_cmd_desc desc;
1199
1200         desc.its_mapd_cmd.dev = dev;
1201         desc.its_mapd_cmd.valid = !!valid;
1202
1203         its_send_single_command(dev->its, its_build_mapd_cmd, &desc);
1204 }
1205
1206 static void its_send_mapc(struct its_node *its, struct its_collection *col,
1207                           int valid)
1208 {
1209         struct its_cmd_desc desc;
1210
1211         desc.its_mapc_cmd.col = col;
1212         desc.its_mapc_cmd.valid = !!valid;
1213
1214         its_send_single_command(its, its_build_mapc_cmd, &desc);
1215 }
1216
1217 static void its_send_mapti(struct its_device *dev, u32 irq_id, u32 id)
1218 {
1219         struct its_cmd_desc desc;
1220
1221         desc.its_mapti_cmd.dev = dev;
1222         desc.its_mapti_cmd.phys_id = irq_id;
1223         desc.its_mapti_cmd.event_id = id;
1224
1225         its_send_single_command(dev->its, its_build_mapti_cmd, &desc);
1226 }
1227
1228 static void its_send_movi(struct its_device *dev,
1229                           struct its_collection *col, u32 id)
1230 {
1231         struct its_cmd_desc desc;
1232
1233         desc.its_movi_cmd.dev = dev;
1234         desc.its_movi_cmd.col = col;
1235         desc.its_movi_cmd.event_id = id;
1236
1237         its_send_single_command(dev->its, its_build_movi_cmd, &desc);
1238 }
1239
1240 static void its_send_discard(struct its_device *dev, u32 id)
1241 {
1242         struct its_cmd_desc desc;
1243
1244         desc.its_discard_cmd.dev = dev;
1245         desc.its_discard_cmd.event_id = id;
1246
1247         its_send_single_command(dev->its, its_build_discard_cmd, &desc);
1248 }
1249
1250 static void its_send_invall(struct its_node *its, struct its_collection *col)
1251 {
1252         struct its_cmd_desc desc;
1253
1254         desc.its_invall_cmd.col = col;
1255
1256         its_send_single_command(its, its_build_invall_cmd, &desc);
1257 }
1258
1259 static void its_send_vmapti(struct its_device *dev, u32 id)
1260 {
1261         struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
1262         struct its_cmd_desc desc;
1263
1264         desc.its_vmapti_cmd.vpe = map->vpe;
1265         desc.its_vmapti_cmd.dev = dev;
1266         desc.its_vmapti_cmd.virt_id = map->vintid;
1267         desc.its_vmapti_cmd.event_id = id;
1268         desc.its_vmapti_cmd.db_enabled = map->db_enabled;
1269
1270         its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc);
1271 }
1272
1273 static void its_send_vmovi(struct its_device *dev, u32 id)
1274 {
1275         struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
1276         struct its_cmd_desc desc;
1277
1278         desc.its_vmovi_cmd.vpe = map->vpe;
1279         desc.its_vmovi_cmd.dev = dev;
1280         desc.its_vmovi_cmd.event_id = id;
1281         desc.its_vmovi_cmd.db_enabled = map->db_enabled;
1282
1283         its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc);
1284 }
1285
1286 static void its_send_vmapp(struct its_node *its,
1287                            struct its_vpe *vpe, bool valid)
1288 {
1289         struct its_cmd_desc desc;
1290
1291         desc.its_vmapp_cmd.vpe = vpe;
1292         desc.its_vmapp_cmd.valid = valid;
1293         desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx];
1294
1295         its_send_single_vcommand(its, its_build_vmapp_cmd, &desc);
1296 }
1297
1298 static void its_send_vmovp(struct its_vpe *vpe)
1299 {
1300         struct its_cmd_desc desc = {};
1301         struct its_node *its;
1302         unsigned long flags;
1303         int col_id = vpe->col_idx;
1304
1305         desc.its_vmovp_cmd.vpe = vpe;
1306
1307         if (!its_list_map) {
1308                 its = list_first_entry(&its_nodes, struct its_node, entry);
1309                 desc.its_vmovp_cmd.col = &its->collections[col_id];
1310                 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
1311                 return;
1312         }
1313
1314         /*
1315          * Yet another marvel of the architecture. If using the
1316          * its_list "feature", we need to make sure that all ITSs
1317          * receive all VMOVP commands in the same order. The only way
1318          * to guarantee this is to make vmovp a serialization point.
1319          *
1320          * Wall <-- Head.
1321          */
1322         raw_spin_lock_irqsave(&vmovp_lock, flags);
1323
1324         desc.its_vmovp_cmd.seq_num = vmovp_seq_num++;
1325         desc.its_vmovp_cmd.its_list = get_its_list(vpe->its_vm);
1326
1327         /* Emit VMOVPs */
1328         list_for_each_entry(its, &its_nodes, entry) {
1329                 if (!is_v4(its))
1330                         continue;
1331
1332                 if (!require_its_list_vmovp(vpe->its_vm, its))
1333                         continue;
1334
1335                 desc.its_vmovp_cmd.col = &its->collections[col_id];
1336                 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
1337         }
1338
1339         raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1340 }
1341
1342 static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe)
1343 {
1344         struct its_cmd_desc desc;
1345
1346         desc.its_vinvall_cmd.vpe = vpe;
1347         its_send_single_vcommand(its, its_build_vinvall_cmd, &desc);
1348 }
1349
1350 static void its_send_vinv(struct its_device *dev, u32 event_id)
1351 {
1352         struct its_cmd_desc desc;
1353
1354         /*
1355          * There is no real VINV command. This is just a normal INV,
1356          * with a VSYNC instead of a SYNC.
1357          */
1358         desc.its_inv_cmd.dev = dev;
1359         desc.its_inv_cmd.event_id = event_id;
1360
1361         its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc);
1362 }
1363
1364 static void its_send_vint(struct its_device *dev, u32 event_id)
1365 {
1366         struct its_cmd_desc desc;
1367
1368         /*
1369          * There is no real VINT command. This is just a normal INT,
1370          * with a VSYNC instead of a SYNC.
1371          */
1372         desc.its_int_cmd.dev = dev;
1373         desc.its_int_cmd.event_id = event_id;
1374
1375         its_send_single_vcommand(dev->its, its_build_vint_cmd, &desc);
1376 }
1377
1378 static void its_send_vclear(struct its_device *dev, u32 event_id)
1379 {
1380         struct its_cmd_desc desc;
1381
1382         /*
1383          * There is no real VCLEAR command. This is just a normal CLEAR,
1384          * with a VSYNC instead of a SYNC.
1385          */
1386         desc.its_clear_cmd.dev = dev;
1387         desc.its_clear_cmd.event_id = event_id;
1388
1389         its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc);
1390 }
1391
1392 static void its_send_invdb(struct its_node *its, struct its_vpe *vpe)
1393 {
1394         struct its_cmd_desc desc;
1395
1396         desc.its_invdb_cmd.vpe = vpe;
1397         its_send_single_vcommand(its, its_build_invdb_cmd, &desc);
1398 }
1399
1400 /*
1401  * irqchip functions - assumes MSI, mostly.
1402  */
1403 static void lpi_write_config(struct irq_data *d, u8 clr, u8 set)
1404 {
1405         struct its_vlpi_map *map = get_vlpi_map(d);
1406         irq_hw_number_t hwirq;
1407         void *va;
1408         u8 *cfg;
1409
1410         if (map) {
1411                 va = page_address(map->vm->vprop_page);
1412                 hwirq = map->vintid;
1413
1414                 /* Remember the updated property */
1415                 map->properties &= ~clr;
1416                 map->properties |= set | LPI_PROP_GROUP1;
1417         } else {
1418                 va = gic_rdists->prop_table_va;
1419                 hwirq = d->hwirq;
1420         }
1421
1422         cfg = va + hwirq - 8192;
1423         *cfg &= ~clr;
1424         *cfg |= set | LPI_PROP_GROUP1;
1425
1426         /*
1427          * Make the above write visible to the redistributors.
1428          * And yes, we're flushing exactly: One. Single. Byte.
1429          * Humpf...
1430          */
1431         if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING)
1432                 gic_flush_dcache_to_poc(cfg, sizeof(*cfg));
1433         else
1434                 dsb(ishst);
1435 }
1436
1437 static void wait_for_syncr(void __iomem *rdbase)
1438 {
1439         while (readl_relaxed(rdbase + GICR_SYNCR) & 1)
1440                 cpu_relax();
1441 }
1442
1443 static void __direct_lpi_inv(struct irq_data *d, u64 val)
1444 {
1445         void __iomem *rdbase;
1446         unsigned long flags;
1447         int cpu;
1448
1449         /* Target the redistributor this LPI is currently routed to */
1450         cpu = irq_to_cpuid_lock(d, &flags);
1451         raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
1452
1453         rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
1454         gic_write_lpir(val, rdbase + GICR_INVLPIR);
1455         wait_for_syncr(rdbase);
1456
1457         raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
1458         irq_to_cpuid_unlock(d, flags);
1459 }
1460
1461 static void direct_lpi_inv(struct irq_data *d)
1462 {
1463         struct its_vlpi_map *map = get_vlpi_map(d);
1464         u64 val;
1465
1466         if (map) {
1467                 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1468
1469                 WARN_ON(!is_v4_1(its_dev->its));
1470
1471                 val  = GICR_INVLPIR_V;
1472                 val |= FIELD_PREP(GICR_INVLPIR_VPEID, map->vpe->vpe_id);
1473                 val |= FIELD_PREP(GICR_INVLPIR_INTID, map->vintid);
1474         } else {
1475                 val = d->hwirq;
1476         }
1477
1478         __direct_lpi_inv(d, val);
1479 }
1480
1481 static void lpi_update_config(struct irq_data *d, u8 clr, u8 set)
1482 {
1483         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1484
1485         lpi_write_config(d, clr, set);
1486         if (gic_rdists->has_direct_lpi &&
1487             (is_v4_1(its_dev->its) || !irqd_is_forwarded_to_vcpu(d)))
1488                 direct_lpi_inv(d);
1489         else if (!irqd_is_forwarded_to_vcpu(d))
1490                 its_send_inv(its_dev, its_get_event_id(d));
1491         else
1492                 its_send_vinv(its_dev, its_get_event_id(d));
1493 }
1494
1495 static void its_vlpi_set_doorbell(struct irq_data *d, bool enable)
1496 {
1497         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1498         u32 event = its_get_event_id(d);
1499         struct its_vlpi_map *map;
1500
1501         /*
1502          * GICv4.1 does away with the per-LPI nonsense, nothing to do
1503          * here.
1504          */
1505         if (is_v4_1(its_dev->its))
1506                 return;
1507
1508         map = dev_event_to_vlpi_map(its_dev, event);
1509
1510         if (map->db_enabled == enable)
1511                 return;
1512
1513         map->db_enabled = enable;
1514
1515         /*
1516          * More fun with the architecture:
1517          *
1518          * Ideally, we'd issue a VMAPTI to set the doorbell to its LPI
1519          * value or to 1023, depending on the enable bit. But that
1520          * would be issuing a mapping for an /existing/ DevID+EventID
1521          * pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI
1522          * to the /same/ vPE, using this opportunity to adjust the
1523          * doorbell. Mouahahahaha. We loves it, Precious.
1524          */
1525         its_send_vmovi(its_dev, event);
1526 }
1527
1528 static void its_mask_irq(struct irq_data *d)
1529 {
1530         if (irqd_is_forwarded_to_vcpu(d))
1531                 its_vlpi_set_doorbell(d, false);
1532
1533         lpi_update_config(d, LPI_PROP_ENABLED, 0);
1534 }
1535
1536 static void its_unmask_irq(struct irq_data *d)
1537 {
1538         if (irqd_is_forwarded_to_vcpu(d))
1539                 its_vlpi_set_doorbell(d, true);
1540
1541         lpi_update_config(d, 0, LPI_PROP_ENABLED);
1542 }
1543
1544 static __maybe_unused u32 its_read_lpi_count(struct irq_data *d, int cpu)
1545 {
1546         if (irqd_affinity_is_managed(d))
1547                 return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1548
1549         return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1550 }
1551
1552 static void its_inc_lpi_count(struct irq_data *d, int cpu)
1553 {
1554         if (irqd_affinity_is_managed(d))
1555                 atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1556         else
1557                 atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1558 }
1559
1560 static void its_dec_lpi_count(struct irq_data *d, int cpu)
1561 {
1562         if (irqd_affinity_is_managed(d))
1563                 atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1564         else
1565                 atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1566 }
1567
1568 static unsigned int cpumask_pick_least_loaded(struct irq_data *d,
1569                                               const struct cpumask *cpu_mask)
1570 {
1571         unsigned int cpu = nr_cpu_ids, tmp;
1572         int count = S32_MAX;
1573
1574         for_each_cpu(tmp, cpu_mask) {
1575                 int this_count = its_read_lpi_count(d, tmp);
1576                 if (this_count < count) {
1577                         cpu = tmp;
1578                         count = this_count;
1579                 }
1580         }
1581
1582         return cpu;
1583 }
1584
1585 /*
1586  * As suggested by Thomas Gleixner in:
1587  * https://lore.kernel.org/r/87h80q2aoc.fsf@nanos.tec.linutronix.de
1588  */
1589 static int its_select_cpu(struct irq_data *d,
1590                           const struct cpumask *aff_mask)
1591 {
1592         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1593         cpumask_var_t tmpmask;
1594         int cpu, node;
1595
1596         if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC))
1597                 return -ENOMEM;
1598
1599         node = its_dev->its->numa_node;
1600
1601         if (!irqd_affinity_is_managed(d)) {
1602                 /* First try the NUMA node */
1603                 if (node != NUMA_NO_NODE) {
1604                         /*
1605                          * Try the intersection of the affinity mask and the
1606                          * node mask (and the online mask, just to be safe).
1607                          */
1608                         cpumask_and(tmpmask, cpumask_of_node(node), aff_mask);
1609                         cpumask_and(tmpmask, tmpmask, cpu_online_mask);
1610
1611                         /*
1612                          * Ideally, we would check if the mask is empty, and
1613                          * try again on the full node here.
1614                          *
1615                          * But it turns out that the way ACPI describes the
1616                          * affinity for ITSs only deals about memory, and
1617                          * not target CPUs, so it cannot describe a single
1618                          * ITS placed next to two NUMA nodes.
1619                          *
1620                          * Instead, just fallback on the online mask. This
1621                          * diverges from Thomas' suggestion above.
1622                          */
1623                         cpu = cpumask_pick_least_loaded(d, tmpmask);
1624                         if (cpu < nr_cpu_ids)
1625                                 goto out;
1626
1627                         /* If we can't cross sockets, give up */
1628                         if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144))
1629                                 goto out;
1630
1631                         /* If the above failed, expand the search */
1632                 }
1633
1634                 /* Try the intersection of the affinity and online masks */
1635                 cpumask_and(tmpmask, aff_mask, cpu_online_mask);
1636
1637                 /* If that doesn't fly, the online mask is the last resort */
1638                 if (cpumask_empty(tmpmask))
1639                         cpumask_copy(tmpmask, cpu_online_mask);
1640
1641                 cpu = cpumask_pick_least_loaded(d, tmpmask);
1642         } else {
1643                 cpumask_copy(tmpmask, aff_mask);
1644
1645                 /* If we cannot cross sockets, limit the search to that node */
1646                 if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) &&
1647                     node != NUMA_NO_NODE)
1648                         cpumask_and(tmpmask, tmpmask, cpumask_of_node(node));
1649
1650                 cpu = cpumask_pick_least_loaded(d, tmpmask);
1651         }
1652 out:
1653         free_cpumask_var(tmpmask);
1654
1655         pr_debug("IRQ%d -> %*pbl CPU%d\n", d->irq, cpumask_pr_args(aff_mask), cpu);
1656         return cpu;
1657 }
1658
1659 static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
1660                             bool force)
1661 {
1662         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1663         struct its_collection *target_col;
1664         u32 id = its_get_event_id(d);
1665         int cpu, prev_cpu;
1666
1667         /* A forwarded interrupt should use irq_set_vcpu_affinity */
1668         if (irqd_is_forwarded_to_vcpu(d))
1669                 return -EINVAL;
1670
1671         prev_cpu = its_dev->event_map.col_map[id];
1672         its_dec_lpi_count(d, prev_cpu);
1673
1674         if (!force)
1675                 cpu = its_select_cpu(d, mask_val);
1676         else
1677                 cpu = cpumask_pick_least_loaded(d, mask_val);
1678
1679         if (cpu < 0 || cpu >= nr_cpu_ids)
1680                 goto err;
1681
1682         /* don't set the affinity when the target cpu is same as current one */
1683         if (cpu != prev_cpu) {
1684                 target_col = &its_dev->its->collections[cpu];
1685                 its_send_movi(its_dev, target_col, id);
1686                 its_dev->event_map.col_map[id] = cpu;
1687                 irq_data_update_effective_affinity(d, cpumask_of(cpu));
1688         }
1689
1690         its_inc_lpi_count(d, cpu);
1691
1692         return IRQ_SET_MASK_OK_DONE;
1693
1694 err:
1695         its_inc_lpi_count(d, prev_cpu);
1696         return -EINVAL;
1697 }
1698
1699 static u64 its_irq_get_msi_base(struct its_device *its_dev)
1700 {
1701         struct its_node *its = its_dev->its;
1702
1703         return its->phys_base + GITS_TRANSLATER;
1704 }
1705
1706 static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
1707 {
1708         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1709         struct its_node *its;
1710         u64 addr;
1711
1712         its = its_dev->its;
1713         addr = its->get_msi_base(its_dev);
1714
1715         msg->address_lo         = lower_32_bits(addr);
1716         msg->address_hi         = upper_32_bits(addr);
1717         msg->data               = its_get_event_id(d);
1718
1719         iommu_dma_compose_msi_msg(irq_data_get_msi_desc(d), msg);
1720 }
1721
1722 static int its_irq_set_irqchip_state(struct irq_data *d,
1723                                      enum irqchip_irq_state which,
1724                                      bool state)
1725 {
1726         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1727         u32 event = its_get_event_id(d);
1728
1729         if (which != IRQCHIP_STATE_PENDING)
1730                 return -EINVAL;
1731
1732         if (irqd_is_forwarded_to_vcpu(d)) {
1733                 if (state)
1734                         its_send_vint(its_dev, event);
1735                 else
1736                         its_send_vclear(its_dev, event);
1737         } else {
1738                 if (state)
1739                         its_send_int(its_dev, event);
1740                 else
1741                         its_send_clear(its_dev, event);
1742         }
1743
1744         return 0;
1745 }
1746
1747 static int its_irq_retrigger(struct irq_data *d)
1748 {
1749         return !its_irq_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
1750 }
1751
1752 /*
1753  * Two favourable cases:
1754  *
1755  * (a) Either we have a GICv4.1, and all vPEs have to be mapped at all times
1756  *     for vSGI delivery
1757  *
1758  * (b) Or the ITSs do not use a list map, meaning that VMOVP is cheap enough
1759  *     and we're better off mapping all VPEs always
1760  *
1761  * If neither (a) nor (b) is true, then we map vPEs on demand.
1762  *
1763  */
1764 static bool gic_requires_eager_mapping(void)
1765 {
1766         if (!its_list_map || gic_rdists->has_rvpeid)
1767                 return true;
1768
1769         return false;
1770 }
1771
1772 static void its_map_vm(struct its_node *its, struct its_vm *vm)
1773 {
1774         unsigned long flags;
1775
1776         if (gic_requires_eager_mapping())
1777                 return;
1778
1779         raw_spin_lock_irqsave(&vmovp_lock, flags);
1780
1781         /*
1782          * If the VM wasn't mapped yet, iterate over the vpes and get
1783          * them mapped now.
1784          */
1785         vm->vlpi_count[its->list_nr]++;
1786
1787         if (vm->vlpi_count[its->list_nr] == 1) {
1788                 int i;
1789
1790                 for (i = 0; i < vm->nr_vpes; i++) {
1791                         struct its_vpe *vpe = vm->vpes[i];
1792                         struct irq_data *d = irq_get_irq_data(vpe->irq);
1793
1794                         /* Map the VPE to the first possible CPU */
1795                         vpe->col_idx = cpumask_first(cpu_online_mask);
1796                         its_send_vmapp(its, vpe, true);
1797                         its_send_vinvall(its, vpe);
1798                         irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
1799                 }
1800         }
1801
1802         raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1803 }
1804
1805 static void its_unmap_vm(struct its_node *its, struct its_vm *vm)
1806 {
1807         unsigned long flags;
1808
1809         /* Not using the ITS list? Everything is always mapped. */
1810         if (gic_requires_eager_mapping())
1811                 return;
1812
1813         raw_spin_lock_irqsave(&vmovp_lock, flags);
1814
1815         if (!--vm->vlpi_count[its->list_nr]) {
1816                 int i;
1817
1818                 for (i = 0; i < vm->nr_vpes; i++)
1819                         its_send_vmapp(its, vm->vpes[i], false);
1820         }
1821
1822         raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1823 }
1824
1825 static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info)
1826 {
1827         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1828         u32 event = its_get_event_id(d);
1829         int ret = 0;
1830
1831         if (!info->map)
1832                 return -EINVAL;
1833
1834         raw_spin_lock(&its_dev->event_map.vlpi_lock);
1835
1836         if (!its_dev->event_map.vm) {
1837                 struct its_vlpi_map *maps;
1838
1839                 maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps),
1840                                GFP_ATOMIC);
1841                 if (!maps) {
1842                         ret = -ENOMEM;
1843                         goto out;
1844                 }
1845
1846                 its_dev->event_map.vm = info->map->vm;
1847                 its_dev->event_map.vlpi_maps = maps;
1848         } else if (its_dev->event_map.vm != info->map->vm) {
1849                 ret = -EINVAL;
1850                 goto out;
1851         }
1852
1853         /* Get our private copy of the mapping information */
1854         its_dev->event_map.vlpi_maps[event] = *info->map;
1855
1856         if (irqd_is_forwarded_to_vcpu(d)) {
1857                 /* Already mapped, move it around */
1858                 its_send_vmovi(its_dev, event);
1859         } else {
1860                 /* Ensure all the VPEs are mapped on this ITS */
1861                 its_map_vm(its_dev->its, info->map->vm);
1862
1863                 /*
1864                  * Flag the interrupt as forwarded so that we can
1865                  * start poking the virtual property table.
1866                  */
1867                 irqd_set_forwarded_to_vcpu(d);
1868
1869                 /* Write out the property to the prop table */
1870                 lpi_write_config(d, 0xff, info->map->properties);
1871
1872                 /* Drop the physical mapping */
1873                 its_send_discard(its_dev, event);
1874
1875                 /* and install the virtual one */
1876                 its_send_vmapti(its_dev, event);
1877
1878                 /* Increment the number of VLPIs */
1879                 its_dev->event_map.nr_vlpis++;
1880         }
1881
1882 out:
1883         raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1884         return ret;
1885 }
1886
1887 static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info)
1888 {
1889         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1890         struct its_vlpi_map *map;
1891         int ret = 0;
1892
1893         raw_spin_lock(&its_dev->event_map.vlpi_lock);
1894
1895         map = get_vlpi_map(d);
1896
1897         if (!its_dev->event_map.vm || !map) {
1898                 ret = -EINVAL;
1899                 goto out;
1900         }
1901
1902         /* Copy our mapping information to the incoming request */
1903         *info->map = *map;
1904
1905 out:
1906         raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1907         return ret;
1908 }
1909
1910 static int its_vlpi_unmap(struct irq_data *d)
1911 {
1912         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1913         u32 event = its_get_event_id(d);
1914         int ret = 0;
1915
1916         raw_spin_lock(&its_dev->event_map.vlpi_lock);
1917
1918         if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) {
1919                 ret = -EINVAL;
1920                 goto out;
1921         }
1922
1923         /* Drop the virtual mapping */
1924         its_send_discard(its_dev, event);
1925
1926         /* and restore the physical one */
1927         irqd_clr_forwarded_to_vcpu(d);
1928         its_send_mapti(its_dev, d->hwirq, event);
1929         lpi_update_config(d, 0xff, (LPI_PROP_DEFAULT_PRIO |
1930                                     LPI_PROP_ENABLED |
1931                                     LPI_PROP_GROUP1));
1932
1933         /* Potentially unmap the VM from this ITS */
1934         its_unmap_vm(its_dev->its, its_dev->event_map.vm);
1935
1936         /*
1937          * Drop the refcount and make the device available again if
1938          * this was the last VLPI.
1939          */
1940         if (!--its_dev->event_map.nr_vlpis) {
1941                 its_dev->event_map.vm = NULL;
1942                 kfree(its_dev->event_map.vlpi_maps);
1943         }
1944
1945 out:
1946         raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1947         return ret;
1948 }
1949
1950 static int its_vlpi_prop_update(struct irq_data *d, struct its_cmd_info *info)
1951 {
1952         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1953
1954         if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d))
1955                 return -EINVAL;
1956
1957         if (info->cmd_type == PROP_UPDATE_AND_INV_VLPI)
1958                 lpi_update_config(d, 0xff, info->config);
1959         else
1960                 lpi_write_config(d, 0xff, info->config);
1961         its_vlpi_set_doorbell(d, !!(info->config & LPI_PROP_ENABLED));
1962
1963         return 0;
1964 }
1965
1966 static int its_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
1967 {
1968         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1969         struct its_cmd_info *info = vcpu_info;
1970
1971         /* Need a v4 ITS */
1972         if (!is_v4(its_dev->its))
1973                 return -EINVAL;
1974
1975         /* Unmap request? */
1976         if (!info)
1977                 return its_vlpi_unmap(d);
1978
1979         switch (info->cmd_type) {
1980         case MAP_VLPI:
1981                 return its_vlpi_map(d, info);
1982
1983         case GET_VLPI:
1984                 return its_vlpi_get(d, info);
1985
1986         case PROP_UPDATE_VLPI:
1987         case PROP_UPDATE_AND_INV_VLPI:
1988                 return its_vlpi_prop_update(d, info);
1989
1990         default:
1991                 return -EINVAL;
1992         }
1993 }
1994
1995 static struct irq_chip its_irq_chip = {
1996         .name                   = "ITS",
1997         .irq_mask               = its_mask_irq,
1998         .irq_unmask             = its_unmask_irq,
1999         .irq_eoi                = irq_chip_eoi_parent,
2000         .irq_set_affinity       = its_set_affinity,
2001         .irq_compose_msi_msg    = its_irq_compose_msi_msg,
2002         .irq_set_irqchip_state  = its_irq_set_irqchip_state,
2003         .irq_retrigger          = its_irq_retrigger,
2004         .irq_set_vcpu_affinity  = its_irq_set_vcpu_affinity,
2005 };
2006
2007
2008 /*
2009  * How we allocate LPIs:
2010  *
2011  * lpi_range_list contains ranges of LPIs that are to available to
2012  * allocate from. To allocate LPIs, just pick the first range that
2013  * fits the required allocation, and reduce it by the required
2014  * amount. Once empty, remove the range from the list.
2015  *
2016  * To free a range of LPIs, add a free range to the list, sort it and
2017  * merge the result if the new range happens to be adjacent to an
2018  * already free block.
2019  *
2020  * The consequence of the above is that allocation is cost is low, but
2021  * freeing is expensive. We assumes that freeing rarely occurs.
2022  */
2023 #define ITS_MAX_LPI_NRBITS      16 /* 64K LPIs */
2024
2025 static DEFINE_MUTEX(lpi_range_lock);
2026 static LIST_HEAD(lpi_range_list);
2027
2028 struct lpi_range {
2029         struct list_head        entry;
2030         u32                     base_id;
2031         u32                     span;
2032 };
2033
2034 static struct lpi_range *mk_lpi_range(u32 base, u32 span)
2035 {
2036         struct lpi_range *range;
2037
2038         range = kmalloc(sizeof(*range), GFP_KERNEL);
2039         if (range) {
2040                 range->base_id = base;
2041                 range->span = span;
2042         }
2043
2044         return range;
2045 }
2046
2047 static int alloc_lpi_range(u32 nr_lpis, u32 *base)
2048 {
2049         struct lpi_range *range, *tmp;
2050         int err = -ENOSPC;
2051
2052         mutex_lock(&lpi_range_lock);
2053
2054         list_for_each_entry_safe(range, tmp, &lpi_range_list, entry) {
2055                 if (range->span >= nr_lpis) {
2056                         *base = range->base_id;
2057                         range->base_id += nr_lpis;
2058                         range->span -= nr_lpis;
2059
2060                         if (range->span == 0) {
2061                                 list_del(&range->entry);
2062                                 kfree(range);
2063                         }
2064
2065                         err = 0;
2066                         break;
2067                 }
2068         }
2069
2070         mutex_unlock(&lpi_range_lock);
2071
2072         pr_debug("ITS: alloc %u:%u\n", *base, nr_lpis);
2073         return err;
2074 }
2075
2076 static void merge_lpi_ranges(struct lpi_range *a, struct lpi_range *b)
2077 {
2078         if (&a->entry == &lpi_range_list || &b->entry == &lpi_range_list)
2079                 return;
2080         if (a->base_id + a->span != b->base_id)
2081                 return;
2082         b->base_id = a->base_id;
2083         b->span += a->span;
2084         list_del(&a->entry);
2085         kfree(a);
2086 }
2087
2088 static int free_lpi_range(u32 base, u32 nr_lpis)
2089 {
2090         struct lpi_range *new, *old;
2091
2092         new = mk_lpi_range(base, nr_lpis);
2093         if (!new)
2094                 return -ENOMEM;
2095
2096         mutex_lock(&lpi_range_lock);
2097
2098         list_for_each_entry_reverse(old, &lpi_range_list, entry) {
2099                 if (old->base_id < base)
2100                         break;
2101         }
2102         /*
2103          * old is the last element with ->base_id smaller than base,
2104          * so new goes right after it. If there are no elements with
2105          * ->base_id smaller than base, &old->entry ends up pointing
2106          * at the head of the list, and inserting new it the start of
2107          * the list is the right thing to do in that case as well.
2108          */
2109         list_add(&new->entry, &old->entry);
2110         /*
2111          * Now check if we can merge with the preceding and/or
2112          * following ranges.
2113          */
2114         merge_lpi_ranges(old, new);
2115         merge_lpi_ranges(new, list_next_entry(new, entry));
2116
2117         mutex_unlock(&lpi_range_lock);
2118         return 0;
2119 }
2120
2121 static int __init its_lpi_init(u32 id_bits)
2122 {
2123         u32 lpis = (1UL << id_bits) - 8192;
2124         u32 numlpis;
2125         int err;
2126
2127         numlpis = 1UL << GICD_TYPER_NUM_LPIS(gic_rdists->gicd_typer);
2128
2129         if (numlpis > 2 && !WARN_ON(numlpis > lpis)) {
2130                 lpis = numlpis;
2131                 pr_info("ITS: Using hypervisor restricted LPI range [%u]\n",
2132                         lpis);
2133         }
2134
2135         /*
2136          * Initializing the allocator is just the same as freeing the
2137          * full range of LPIs.
2138          */
2139         err = free_lpi_range(8192, lpis);
2140         pr_debug("ITS: Allocator initialized for %u LPIs\n", lpis);
2141         return err;
2142 }
2143
2144 static unsigned long *its_lpi_alloc(int nr_irqs, u32 *base, int *nr_ids)
2145 {
2146         unsigned long *bitmap = NULL;
2147         int err = 0;
2148
2149         do {
2150                 err = alloc_lpi_range(nr_irqs, base);
2151                 if (!err)
2152                         break;
2153
2154                 nr_irqs /= 2;
2155         } while (nr_irqs > 0);
2156
2157         if (!nr_irqs)
2158                 err = -ENOSPC;
2159
2160         if (err)
2161                 goto out;
2162
2163         bitmap = kcalloc(BITS_TO_LONGS(nr_irqs), sizeof (long), GFP_ATOMIC);
2164         if (!bitmap)
2165                 goto out;
2166
2167         *nr_ids = nr_irqs;
2168
2169 out:
2170         if (!bitmap)
2171                 *base = *nr_ids = 0;
2172
2173         return bitmap;
2174 }
2175
2176 static void its_lpi_free(unsigned long *bitmap, u32 base, u32 nr_ids)
2177 {
2178         WARN_ON(free_lpi_range(base, nr_ids));
2179         kfree(bitmap);
2180 }
2181
2182 static void gic_reset_prop_table(void *va)
2183 {
2184         /* Priority 0xa0, Group-1, disabled */
2185         memset(va, LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1, LPI_PROPBASE_SZ);
2186
2187         /* Make sure the GIC will observe the written configuration */
2188         gic_flush_dcache_to_poc(va, LPI_PROPBASE_SZ);
2189 }
2190
2191 static struct page *its_allocate_prop_table(gfp_t gfp_flags)
2192 {
2193         struct page *prop_page;
2194
2195         prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ));
2196         if (!prop_page)
2197                 return NULL;
2198
2199         gic_reset_prop_table(page_address(prop_page));
2200
2201         return prop_page;
2202 }
2203
2204 static void its_free_prop_table(struct page *prop_page)
2205 {
2206         free_pages((unsigned long)page_address(prop_page),
2207                    get_order(LPI_PROPBASE_SZ));
2208 }
2209
2210 static bool gic_check_reserved_range(phys_addr_t addr, unsigned long size)
2211 {
2212         phys_addr_t start, end, addr_end;
2213         u64 i;
2214
2215         /*
2216          * We don't bother checking for a kdump kernel as by
2217          * construction, the LPI tables are out of this kernel's
2218          * memory map.
2219          */
2220         if (is_kdump_kernel())
2221                 return true;
2222
2223         addr_end = addr + size - 1;
2224
2225         for_each_reserved_mem_range(i, &start, &end) {
2226                 if (addr >= start && addr_end <= end)
2227                         return true;
2228         }
2229
2230         /* Not found, not a good sign... */
2231         pr_warn("GICv3: Expected reserved range [%pa:%pa], not found\n",
2232                 &addr, &addr_end);
2233         add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
2234         return false;
2235 }
2236
2237 static int gic_reserve_range(phys_addr_t addr, unsigned long size)
2238 {
2239         if (efi_enabled(EFI_CONFIG_TABLES))
2240                 return efi_mem_reserve_persistent(addr, size);
2241
2242         return 0;
2243 }
2244
2245 static int __init its_setup_lpi_prop_table(void)
2246 {
2247         if (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) {
2248                 u64 val;
2249
2250                 val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
2251                 lpi_id_bits = (val & GICR_PROPBASER_IDBITS_MASK) + 1;
2252
2253                 gic_rdists->prop_table_pa = val & GENMASK_ULL(51, 12);
2254                 gic_rdists->prop_table_va = memremap(gic_rdists->prop_table_pa,
2255                                                      LPI_PROPBASE_SZ,
2256                                                      MEMREMAP_WB);
2257                 gic_reset_prop_table(gic_rdists->prop_table_va);
2258         } else {
2259                 struct page *page;
2260
2261                 lpi_id_bits = min_t(u32,
2262                                     GICD_TYPER_ID_BITS(gic_rdists->gicd_typer),
2263                                     ITS_MAX_LPI_NRBITS);
2264                 page = its_allocate_prop_table(GFP_NOWAIT);
2265                 if (!page) {
2266                         pr_err("Failed to allocate PROPBASE\n");
2267                         return -ENOMEM;
2268                 }
2269
2270                 gic_rdists->prop_table_pa = page_to_phys(page);
2271                 gic_rdists->prop_table_va = page_address(page);
2272                 WARN_ON(gic_reserve_range(gic_rdists->prop_table_pa,
2273                                           LPI_PROPBASE_SZ));
2274         }
2275
2276         pr_info("GICv3: using LPI property table @%pa\n",
2277                 &gic_rdists->prop_table_pa);
2278
2279         return its_lpi_init(lpi_id_bits);
2280 }
2281
2282 static const char *its_base_type_string[] = {
2283         [GITS_BASER_TYPE_DEVICE]        = "Devices",
2284         [GITS_BASER_TYPE_VCPU]          = "Virtual CPUs",
2285         [GITS_BASER_TYPE_RESERVED3]     = "Reserved (3)",
2286         [GITS_BASER_TYPE_COLLECTION]    = "Interrupt Collections",
2287         [GITS_BASER_TYPE_RESERVED5]     = "Reserved (5)",
2288         [GITS_BASER_TYPE_RESERVED6]     = "Reserved (6)",
2289         [GITS_BASER_TYPE_RESERVED7]     = "Reserved (7)",
2290 };
2291
2292 static u64 its_read_baser(struct its_node *its, struct its_baser *baser)
2293 {
2294         u32 idx = baser - its->tables;
2295
2296         return gits_read_baser(its->base + GITS_BASER + (idx << 3));
2297 }
2298
2299 static void its_write_baser(struct its_node *its, struct its_baser *baser,
2300                             u64 val)
2301 {
2302         u32 idx = baser - its->tables;
2303
2304         gits_write_baser(val, its->base + GITS_BASER + (idx << 3));
2305         baser->val = its_read_baser(its, baser);
2306 }
2307
2308 static int its_setup_baser(struct its_node *its, struct its_baser *baser,
2309                            u64 cache, u64 shr, u32 order, bool indirect)
2310 {
2311         u64 val = its_read_baser(its, baser);
2312         u64 esz = GITS_BASER_ENTRY_SIZE(val);
2313         u64 type = GITS_BASER_TYPE(val);
2314         u64 baser_phys, tmp;
2315         u32 alloc_pages, psz;
2316         struct page *page;
2317         void *base;
2318
2319         psz = baser->psz;
2320         alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz);
2321         if (alloc_pages > GITS_BASER_PAGES_MAX) {
2322                 pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n",
2323                         &its->phys_base, its_base_type_string[type],
2324                         alloc_pages, GITS_BASER_PAGES_MAX);
2325                 alloc_pages = GITS_BASER_PAGES_MAX;
2326                 order = get_order(GITS_BASER_PAGES_MAX * psz);
2327         }
2328
2329         page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order);
2330         if (!page)
2331                 return -ENOMEM;
2332
2333         base = (void *)page_address(page);
2334         baser_phys = virt_to_phys(base);
2335
2336         /* Check if the physical address of the memory is above 48bits */
2337         if (IS_ENABLED(CONFIG_ARM64_64K_PAGES) && (baser_phys >> 48)) {
2338
2339                 /* 52bit PA is supported only when PageSize=64K */
2340                 if (psz != SZ_64K) {
2341                         pr_err("ITS: no 52bit PA support when psz=%d\n", psz);
2342                         free_pages((unsigned long)base, order);
2343                         return -ENXIO;
2344                 }
2345
2346                 /* Convert 52bit PA to 48bit field */
2347                 baser_phys = GITS_BASER_PHYS_52_to_48(baser_phys);
2348         }
2349
2350 retry_baser:
2351         val = (baser_phys                                        |
2352                 (type << GITS_BASER_TYPE_SHIFT)                  |
2353                 ((esz - 1) << GITS_BASER_ENTRY_SIZE_SHIFT)       |
2354                 ((alloc_pages - 1) << GITS_BASER_PAGES_SHIFT)    |
2355                 cache                                            |
2356                 shr                                              |
2357                 GITS_BASER_VALID);
2358
2359         val |=  indirect ? GITS_BASER_INDIRECT : 0x0;
2360
2361         switch (psz) {
2362         case SZ_4K:
2363                 val |= GITS_BASER_PAGE_SIZE_4K;
2364                 break;
2365         case SZ_16K:
2366                 val |= GITS_BASER_PAGE_SIZE_16K;
2367                 break;
2368         case SZ_64K:
2369                 val |= GITS_BASER_PAGE_SIZE_64K;
2370                 break;
2371         }
2372
2373         its_write_baser(its, baser, val);
2374         tmp = baser->val;
2375
2376         if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) {
2377                 /*
2378                  * Shareability didn't stick. Just use
2379                  * whatever the read reported, which is likely
2380                  * to be the only thing this redistributor
2381                  * supports. If that's zero, make it
2382                  * non-cacheable as well.
2383                  */
2384                 shr = tmp & GITS_BASER_SHAREABILITY_MASK;
2385                 if (!shr) {
2386                         cache = GITS_BASER_nC;
2387                         gic_flush_dcache_to_poc(base, PAGE_ORDER_TO_SIZE(order));
2388                 }
2389                 goto retry_baser;
2390         }
2391
2392         if (val != tmp) {
2393                 pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n",
2394                        &its->phys_base, its_base_type_string[type],
2395                        val, tmp);
2396                 free_pages((unsigned long)base, order);
2397                 return -ENXIO;
2398         }
2399
2400         baser->order = order;
2401         baser->base = base;
2402         baser->psz = psz;
2403         tmp = indirect ? GITS_LVL1_ENTRY_SIZE : esz;
2404
2405         pr_info("ITS@%pa: allocated %d %s @%lx (%s, esz %d, psz %dK, shr %d)\n",
2406                 &its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / (int)tmp),
2407                 its_base_type_string[type],
2408                 (unsigned long)virt_to_phys(base),
2409                 indirect ? "indirect" : "flat", (int)esz,
2410                 psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT);
2411
2412         return 0;
2413 }
2414
2415 static bool its_parse_indirect_baser(struct its_node *its,
2416                                      struct its_baser *baser,
2417                                      u32 *order, u32 ids)
2418 {
2419         u64 tmp = its_read_baser(its, baser);
2420         u64 type = GITS_BASER_TYPE(tmp);
2421         u64 esz = GITS_BASER_ENTRY_SIZE(tmp);
2422         u64 val = GITS_BASER_InnerShareable | GITS_BASER_RaWaWb;
2423         u32 new_order = *order;
2424         u32 psz = baser->psz;
2425         bool indirect = false;
2426
2427         /* No need to enable Indirection if memory requirement < (psz*2)bytes */
2428         if ((esz << ids) > (psz * 2)) {
2429                 /*
2430                  * Find out whether hw supports a single or two-level table by
2431                  * table by reading bit at offset '62' after writing '1' to it.
2432                  */
2433                 its_write_baser(its, baser, val | GITS_BASER_INDIRECT);
2434                 indirect = !!(baser->val & GITS_BASER_INDIRECT);
2435
2436                 if (indirect) {
2437                         /*
2438                          * The size of the lvl2 table is equal to ITS page size
2439                          * which is 'psz'. For computing lvl1 table size,
2440                          * subtract ID bits that sparse lvl2 table from 'ids'
2441                          * which is reported by ITS hardware times lvl1 table
2442                          * entry size.
2443                          */
2444                         ids -= ilog2(psz / (int)esz);
2445                         esz = GITS_LVL1_ENTRY_SIZE;
2446                 }
2447         }
2448
2449         /*
2450          * Allocate as many entries as required to fit the
2451          * range of device IDs that the ITS can grok... The ID
2452          * space being incredibly sparse, this results in a
2453          * massive waste of memory if two-level device table
2454          * feature is not supported by hardware.
2455          */
2456         new_order = max_t(u32, get_order(esz << ids), new_order);
2457         if (new_order >= MAX_ORDER) {
2458                 new_order = MAX_ORDER - 1;
2459                 ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz);
2460                 pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n",
2461                         &its->phys_base, its_base_type_string[type],
2462                         device_ids(its), ids);
2463         }
2464
2465         *order = new_order;
2466
2467         return indirect;
2468 }
2469
2470 static u32 compute_common_aff(u64 val)
2471 {
2472         u32 aff, clpiaff;
2473
2474         aff = FIELD_GET(GICR_TYPER_AFFINITY, val);
2475         clpiaff = FIELD_GET(GICR_TYPER_COMMON_LPI_AFF, val);
2476
2477         return aff & ~(GENMASK(31, 0) >> (clpiaff * 8));
2478 }
2479
2480 static u32 compute_its_aff(struct its_node *its)
2481 {
2482         u64 val;
2483         u32 svpet;
2484
2485         /*
2486          * Reencode the ITS SVPET and MPIDR as a GICR_TYPER, and compute
2487          * the resulting affinity. We then use that to see if this match
2488          * our own affinity.
2489          */
2490         svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);
2491         val  = FIELD_PREP(GICR_TYPER_COMMON_LPI_AFF, svpet);
2492         val |= FIELD_PREP(GICR_TYPER_AFFINITY, its->mpidr);
2493         return compute_common_aff(val);
2494 }
2495
2496 static struct its_node *find_sibling_its(struct its_node *cur_its)
2497 {
2498         struct its_node *its;
2499         u32 aff;
2500
2501         if (!FIELD_GET(GITS_TYPER_SVPET, cur_its->typer))
2502                 return NULL;
2503
2504         aff = compute_its_aff(cur_its);
2505
2506         list_for_each_entry(its, &its_nodes, entry) {
2507                 u64 baser;
2508
2509                 if (!is_v4_1(its) || its == cur_its)
2510                         continue;
2511
2512                 if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
2513                         continue;
2514
2515                 if (aff != compute_its_aff(its))
2516                         continue;
2517
2518                 /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
2519                 baser = its->tables[2].val;
2520                 if (!(baser & GITS_BASER_VALID))
2521                         continue;
2522
2523                 return its;
2524         }
2525
2526         return NULL;
2527 }
2528
2529 static void its_free_tables(struct its_node *its)
2530 {
2531         int i;
2532
2533         for (i = 0; i < GITS_BASER_NR_REGS; i++) {
2534                 if (its->tables[i].base) {
2535                         free_pages((unsigned long)its->tables[i].base,
2536                                    its->tables[i].order);
2537                         its->tables[i].base = NULL;
2538                 }
2539         }
2540 }
2541
2542 static int its_probe_baser_psz(struct its_node *its, struct its_baser *baser)
2543 {
2544         u64 psz = SZ_64K;
2545
2546         while (psz) {
2547                 u64 val, gpsz;
2548
2549                 val = its_read_baser(its, baser);
2550                 val &= ~GITS_BASER_PAGE_SIZE_MASK;
2551
2552                 switch (psz) {
2553                 case SZ_64K:
2554                         gpsz = GITS_BASER_PAGE_SIZE_64K;
2555                         break;
2556                 case SZ_16K:
2557                         gpsz = GITS_BASER_PAGE_SIZE_16K;
2558                         break;
2559                 case SZ_4K:
2560                 default:
2561                         gpsz = GITS_BASER_PAGE_SIZE_4K;
2562                         break;
2563                 }
2564
2565                 gpsz >>= GITS_BASER_PAGE_SIZE_SHIFT;
2566
2567                 val |= FIELD_PREP(GITS_BASER_PAGE_SIZE_MASK, gpsz);
2568                 its_write_baser(its, baser, val);
2569
2570                 if (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser->val) == gpsz)
2571                         break;
2572
2573                 switch (psz) {
2574                 case SZ_64K:
2575                         psz = SZ_16K;
2576                         break;
2577                 case SZ_16K:
2578                         psz = SZ_4K;
2579                         break;
2580                 case SZ_4K:
2581                 default:
2582                         return -1;
2583                 }
2584         }
2585
2586         baser->psz = psz;
2587         return 0;
2588 }
2589
2590 static int its_alloc_tables(struct its_node *its)
2591 {
2592         u64 shr = GITS_BASER_InnerShareable;
2593         u64 cache = GITS_BASER_RaWaWb;
2594         int err, i;
2595
2596         if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375)
2597                 /* erratum 24313: ignore memory access type */
2598                 cache = GITS_BASER_nCnB;
2599
2600         for (i = 0; i < GITS_BASER_NR_REGS; i++) {
2601                 struct its_baser *baser = its->tables + i;
2602                 u64 val = its_read_baser(its, baser);
2603                 u64 type = GITS_BASER_TYPE(val);
2604                 bool indirect = false;
2605                 u32 order;
2606
2607                 if (type == GITS_BASER_TYPE_NONE)
2608                         continue;
2609
2610                 if (its_probe_baser_psz(its, baser)) {
2611                         its_free_tables(its);
2612                         return -ENXIO;
2613                 }
2614
2615                 order = get_order(baser->psz);
2616
2617                 switch (type) {
2618                 case GITS_BASER_TYPE_DEVICE:
2619                         indirect = its_parse_indirect_baser(its, baser, &order,
2620                                                             device_ids(its));
2621                         break;
2622
2623                 case GITS_BASER_TYPE_VCPU:
2624                         if (is_v4_1(its)) {
2625                                 struct its_node *sibling;
2626
2627                                 WARN_ON(i != 2);
2628                                 if ((sibling = find_sibling_its(its))) {
2629                                         *baser = sibling->tables[2];
2630                                         its_write_baser(its, baser, baser->val);
2631                                         continue;
2632                                 }
2633                         }
2634
2635                         indirect = its_parse_indirect_baser(its, baser, &order,
2636                                                             ITS_MAX_VPEID_BITS);
2637                         break;
2638                 }
2639
2640                 err = its_setup_baser(its, baser, cache, shr, order, indirect);
2641                 if (err < 0) {
2642                         its_free_tables(its);
2643                         return err;
2644                 }
2645
2646                 /* Update settings which will be used for next BASERn */
2647                 cache = baser->val & GITS_BASER_CACHEABILITY_MASK;
2648                 shr = baser->val & GITS_BASER_SHAREABILITY_MASK;
2649         }
2650
2651         return 0;
2652 }
2653
2654 static u64 inherit_vpe_l1_table_from_its(void)
2655 {
2656         struct its_node *its;
2657         u64 val;
2658         u32 aff;
2659
2660         val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
2661         aff = compute_common_aff(val);
2662
2663         list_for_each_entry(its, &its_nodes, entry) {
2664                 u64 baser, addr;
2665
2666                 if (!is_v4_1(its))
2667                         continue;
2668
2669                 if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
2670                         continue;
2671
2672                 if (aff != compute_its_aff(its))
2673                         continue;
2674
2675                 /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
2676                 baser = its->tables[2].val;
2677                 if (!(baser & GITS_BASER_VALID))
2678                         continue;
2679
2680                 /* We have a winner! */
2681                 gic_data_rdist()->vpe_l1_base = its->tables[2].base;
2682
2683                 val  = GICR_VPROPBASER_4_1_VALID;
2684                 if (baser & GITS_BASER_INDIRECT)
2685                         val |= GICR_VPROPBASER_4_1_INDIRECT;
2686                 val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE,
2687                                   FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser));
2688                 switch (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser)) {
2689                 case GIC_PAGE_SIZE_64K:
2690                         addr = GITS_BASER_ADDR_48_to_52(baser);
2691                         break;
2692                 default:
2693                         addr = baser & GENMASK_ULL(47, 12);
2694                         break;
2695                 }
2696                 val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, addr >> 12);
2697                 val |= FIELD_PREP(GICR_VPROPBASER_SHAREABILITY_MASK,
2698                                   FIELD_GET(GITS_BASER_SHAREABILITY_MASK, baser));
2699                 val |= FIELD_PREP(GICR_VPROPBASER_INNER_CACHEABILITY_MASK,
2700                                   FIELD_GET(GITS_BASER_INNER_CACHEABILITY_MASK, baser));
2701                 val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, GITS_BASER_NR_PAGES(baser) - 1);
2702
2703                 return val;
2704         }
2705
2706         return 0;
2707 }
2708
2709 static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask)
2710 {
2711         u32 aff;
2712         u64 val;
2713         int cpu;
2714
2715         val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
2716         aff = compute_common_aff(val);
2717
2718         for_each_possible_cpu(cpu) {
2719                 void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
2720
2721                 if (!base || cpu == smp_processor_id())
2722                         continue;
2723
2724                 val = gic_read_typer(base + GICR_TYPER);
2725                 if (aff != compute_common_aff(val))
2726                         continue;
2727
2728                 /*
2729                  * At this point, we have a victim. This particular CPU
2730                  * has already booted, and has an affinity that matches
2731                  * ours wrt CommonLPIAff. Let's use its own VPROPBASER.
2732                  * Make sure we don't write the Z bit in that case.
2733                  */
2734                 val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
2735                 val &= ~GICR_VPROPBASER_4_1_Z;
2736
2737                 gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base;
2738                 *mask = gic_data_rdist_cpu(cpu)->vpe_table_mask;
2739
2740                 return val;
2741         }
2742
2743         return 0;
2744 }
2745
2746 static bool allocate_vpe_l2_table(int cpu, u32 id)
2747 {
2748         void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
2749         unsigned int psz, esz, idx, npg, gpsz;
2750         u64 val;
2751         struct page *page;
2752         __le64 *table;
2753
2754         if (!gic_rdists->has_rvpeid)
2755                 return true;
2756
2757         /* Skip non-present CPUs */
2758         if (!base)
2759                 return true;
2760
2761         val  = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
2762
2763         esz  = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val) + 1;
2764         gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
2765         npg  = FIELD_GET(GICR_VPROPBASER_4_1_SIZE, val) + 1;
2766
2767         switch (gpsz) {
2768         default:
2769                 WARN_ON(1);
2770                 fallthrough;
2771         case GIC_PAGE_SIZE_4K:
2772                 psz = SZ_4K;
2773                 break;
2774         case GIC_PAGE_SIZE_16K:
2775                 psz = SZ_16K;
2776                 break;
2777         case GIC_PAGE_SIZE_64K:
2778                 psz = SZ_64K;
2779                 break;
2780         }
2781
2782         /* Don't allow vpe_id that exceeds single, flat table limit */
2783         if (!(val & GICR_VPROPBASER_4_1_INDIRECT))
2784                 return (id < (npg * psz / (esz * SZ_8)));
2785
2786         /* Compute 1st level table index & check if that exceeds table limit */
2787         idx = id >> ilog2(psz / (esz * SZ_8));
2788         if (idx >= (npg * psz / GITS_LVL1_ENTRY_SIZE))
2789                 return false;
2790
2791         table = gic_data_rdist_cpu(cpu)->vpe_l1_base;
2792
2793         /* Allocate memory for 2nd level table */
2794         if (!table[idx]) {
2795                 page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz));
2796                 if (!page)
2797                         return false;
2798
2799                 /* Flush Lvl2 table to PoC if hw doesn't support coherency */
2800                 if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
2801                         gic_flush_dcache_to_poc(page_address(page), psz);
2802
2803                 table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
2804
2805                 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
2806                 if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
2807                         gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
2808
2809                 /* Ensure updated table contents are visible to RD hardware */
2810                 dsb(sy);
2811         }
2812
2813         return true;
2814 }
2815
2816 static int allocate_vpe_l1_table(void)
2817 {
2818         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
2819         u64 val, gpsz, npg, pa;
2820         unsigned int psz = SZ_64K;
2821         unsigned int np, epp, esz;
2822         struct page *page;
2823
2824         if (!gic_rdists->has_rvpeid)
2825                 return 0;
2826
2827         /*
2828          * if VPENDBASER.Valid is set, disable any previously programmed
2829          * VPE by setting PendingLast while clearing Valid. This has the
2830          * effect of making sure no doorbell will be generated and we can
2831          * then safely clear VPROPBASER.Valid.
2832          */
2833         if (gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER) & GICR_VPENDBASER_Valid)
2834                 gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast,
2835                                       vlpi_base + GICR_VPENDBASER);
2836
2837         /*
2838          * If we can inherit the configuration from another RD, let's do
2839          * so. Otherwise, we have to go through the allocation process. We
2840          * assume that all RDs have the exact same requirements, as
2841          * nothing will work otherwise.
2842          */
2843         val = inherit_vpe_l1_table_from_rd(&gic_data_rdist()->vpe_table_mask);
2844         if (val & GICR_VPROPBASER_4_1_VALID)
2845                 goto out;
2846
2847         gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_ATOMIC);
2848         if (!gic_data_rdist()->vpe_table_mask)
2849                 return -ENOMEM;
2850
2851         val = inherit_vpe_l1_table_from_its();
2852         if (val & GICR_VPROPBASER_4_1_VALID)
2853                 goto out;
2854
2855         /* First probe the page size */
2856         val = FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, GIC_PAGE_SIZE_64K);
2857         gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2858         val = gicr_read_vpropbaser(vlpi_base + GICR_VPROPBASER);
2859         gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
2860         esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val);
2861
2862         switch (gpsz) {
2863         default:
2864                 gpsz = GIC_PAGE_SIZE_4K;
2865                 fallthrough;
2866         case GIC_PAGE_SIZE_4K:
2867                 psz = SZ_4K;
2868                 break;
2869         case GIC_PAGE_SIZE_16K:
2870                 psz = SZ_16K;
2871                 break;
2872         case GIC_PAGE_SIZE_64K:
2873                 psz = SZ_64K;
2874                 break;
2875         }
2876
2877         /*
2878          * Start populating the register from scratch, including RO fields
2879          * (which we want to print in debug cases...)
2880          */
2881         val = 0;
2882         val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, gpsz);
2883         val |= FIELD_PREP(GICR_VPROPBASER_4_1_ENTRY_SIZE, esz);
2884
2885         /* How many entries per GIC page? */
2886         esz++;
2887         epp = psz / (esz * SZ_8);
2888
2889         /*
2890          * If we need more than just a single L1 page, flag the table
2891          * as indirect and compute the number of required L1 pages.
2892          */
2893         if (epp < ITS_MAX_VPEID) {
2894                 int nl2;
2895
2896                 val |= GICR_VPROPBASER_4_1_INDIRECT;
2897
2898                 /* Number of L2 pages required to cover the VPEID space */
2899                 nl2 = DIV_ROUND_UP(ITS_MAX_VPEID, epp);
2900
2901                 /* Number of L1 pages to point to the L2 pages */
2902                 npg = DIV_ROUND_UP(nl2 * SZ_8, psz);
2903         } else {
2904                 npg = 1;
2905         }
2906
2907         val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg - 1);
2908
2909         /* Right, that's the number of CPU pages we need for L1 */
2910         np = DIV_ROUND_UP(npg * psz, PAGE_SIZE);
2911
2912         pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n",
2913                  np, npg, psz, epp, esz);
2914         page = alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE));
2915         if (!page)
2916                 return -ENOMEM;
2917
2918         gic_data_rdist()->vpe_l1_base = page_address(page);
2919         pa = virt_to_phys(page_address(page));
2920         WARN_ON(!IS_ALIGNED(pa, psz));
2921
2922         val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, pa >> 12);
2923         val |= GICR_VPROPBASER_RaWb;
2924         val |= GICR_VPROPBASER_InnerShareable;
2925         val |= GICR_VPROPBASER_4_1_Z;
2926         val |= GICR_VPROPBASER_4_1_VALID;
2927
2928 out:
2929         gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2930         cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask);
2931
2932         pr_debug("CPU%d: VPROPBASER = %llx %*pbl\n",
2933                  smp_processor_id(), val,
2934                  cpumask_pr_args(gic_data_rdist()->vpe_table_mask));
2935
2936         return 0;
2937 }
2938
2939 static int its_alloc_collections(struct its_node *its)
2940 {
2941         int i;
2942
2943         its->collections = kcalloc(nr_cpu_ids, sizeof(*its->collections),
2944                                    GFP_KERNEL);
2945         if (!its->collections)
2946                 return -ENOMEM;
2947
2948         for (i = 0; i < nr_cpu_ids; i++)
2949                 its->collections[i].target_address = ~0ULL;
2950
2951         return 0;
2952 }
2953
2954 static struct page *its_allocate_pending_table(gfp_t gfp_flags)
2955 {
2956         struct page *pend_page;
2957
2958         pend_page = alloc_pages(gfp_flags | __GFP_ZERO,
2959                                 get_order(LPI_PENDBASE_SZ));
2960         if (!pend_page)
2961                 return NULL;
2962
2963         /* Make sure the GIC will observe the zero-ed page */
2964         gic_flush_dcache_to_poc(page_address(pend_page), LPI_PENDBASE_SZ);
2965
2966         return pend_page;
2967 }
2968
2969 static void its_free_pending_table(struct page *pt)
2970 {
2971         free_pages((unsigned long)page_address(pt), get_order(LPI_PENDBASE_SZ));
2972 }
2973
2974 /*
2975  * Booting with kdump and LPIs enabled is generally fine. Any other
2976  * case is wrong in the absence of firmware/EFI support.
2977  */
2978 static bool enabled_lpis_allowed(void)
2979 {
2980         phys_addr_t addr;
2981         u64 val;
2982
2983         /* Check whether the property table is in a reserved region */
2984         val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
2985         addr = val & GENMASK_ULL(51, 12);
2986
2987         return gic_check_reserved_range(addr, LPI_PROPBASE_SZ);
2988 }
2989
2990 static int __init allocate_lpi_tables(void)
2991 {
2992         u64 val;
2993         int err, cpu;
2994
2995         /*
2996          * If LPIs are enabled while we run this from the boot CPU,
2997          * flag the RD tables as pre-allocated if the stars do align.
2998          */
2999         val = readl_relaxed(gic_data_rdist_rd_base() + GICR_CTLR);
3000         if ((val & GICR_CTLR_ENABLE_LPIS) && enabled_lpis_allowed()) {
3001                 gic_rdists->flags |= (RDIST_FLAGS_RD_TABLES_PREALLOCATED |
3002                                       RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING);
3003                 pr_info("GICv3: Using preallocated redistributor tables\n");
3004         }
3005
3006         err = its_setup_lpi_prop_table();
3007         if (err)
3008                 return err;
3009
3010         /*
3011          * We allocate all the pending tables anyway, as we may have a
3012          * mix of RDs that have had LPIs enabled, and some that
3013          * don't. We'll free the unused ones as each CPU comes online.
3014          */
3015         for_each_possible_cpu(cpu) {
3016                 struct page *pend_page;
3017
3018                 pend_page = its_allocate_pending_table(GFP_NOWAIT);
3019                 if (!pend_page) {
3020                         pr_err("Failed to allocate PENDBASE for CPU%d\n", cpu);
3021                         return -ENOMEM;
3022                 }
3023
3024                 gic_data_rdist_cpu(cpu)->pend_page = pend_page;
3025         }
3026
3027         return 0;
3028 }
3029
3030 static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set)
3031 {
3032         u32 count = 1000000;    /* 1s! */
3033         bool clean;
3034         u64 val;
3035
3036         val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
3037         val &= ~GICR_VPENDBASER_Valid;
3038         val &= ~clr;
3039         val |= set;
3040         gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
3041
3042         do {
3043                 val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
3044                 clean = !(val & GICR_VPENDBASER_Dirty);
3045                 if (!clean) {
3046                         count--;
3047                         cpu_relax();
3048                         udelay(1);
3049                 }
3050         } while (!clean && count);
3051
3052         if (unlikely(val & GICR_VPENDBASER_Dirty)) {
3053                 pr_err_ratelimited("ITS virtual pending table not cleaning\n");
3054                 val |= GICR_VPENDBASER_PendingLast;
3055         }
3056
3057         return val;
3058 }
3059
3060 static void its_cpu_init_lpis(void)
3061 {
3062         void __iomem *rbase = gic_data_rdist_rd_base();
3063         struct page *pend_page;
3064         phys_addr_t paddr;
3065         u64 val, tmp;
3066
3067         if (gic_data_rdist()->lpi_enabled)
3068                 return;
3069
3070         val = readl_relaxed(rbase + GICR_CTLR);
3071         if ((gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) &&
3072             (val & GICR_CTLR_ENABLE_LPIS)) {
3073                 /*
3074                  * Check that we get the same property table on all
3075                  * RDs. If we don't, this is hopeless.
3076                  */
3077                 paddr = gicr_read_propbaser(rbase + GICR_PROPBASER);
3078                 paddr &= GENMASK_ULL(51, 12);
3079                 if (WARN_ON(gic_rdists->prop_table_pa != paddr))
3080                         add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
3081
3082                 paddr = gicr_read_pendbaser(rbase + GICR_PENDBASER);
3083                 paddr &= GENMASK_ULL(51, 16);
3084
3085                 WARN_ON(!gic_check_reserved_range(paddr, LPI_PENDBASE_SZ));
3086                 its_free_pending_table(gic_data_rdist()->pend_page);
3087                 gic_data_rdist()->pend_page = NULL;
3088
3089                 goto out;
3090         }
3091
3092         pend_page = gic_data_rdist()->pend_page;
3093         paddr = page_to_phys(pend_page);
3094         WARN_ON(gic_reserve_range(paddr, LPI_PENDBASE_SZ));
3095
3096         /* set PROPBASE */
3097         val = (gic_rdists->prop_table_pa |
3098                GICR_PROPBASER_InnerShareable |
3099                GICR_PROPBASER_RaWaWb |
3100                ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK));
3101
3102         gicr_write_propbaser(val, rbase + GICR_PROPBASER);
3103         tmp = gicr_read_propbaser(rbase + GICR_PROPBASER);
3104
3105         if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) {
3106                 if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) {
3107                         /*
3108                          * The HW reports non-shareable, we must
3109                          * remove the cacheability attributes as
3110                          * well.
3111                          */
3112                         val &= ~(GICR_PROPBASER_SHAREABILITY_MASK |
3113                                  GICR_PROPBASER_CACHEABILITY_MASK);
3114                         val |= GICR_PROPBASER_nC;
3115                         gicr_write_propbaser(val, rbase + GICR_PROPBASER);
3116                 }
3117                 pr_info_once("GIC: using cache flushing for LPI property table\n");
3118                 gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING;
3119         }
3120
3121         /* set PENDBASE */
3122         val = (page_to_phys(pend_page) |
3123                GICR_PENDBASER_InnerShareable |
3124                GICR_PENDBASER_RaWaWb);
3125
3126         gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
3127         tmp = gicr_read_pendbaser(rbase + GICR_PENDBASER);
3128
3129         if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) {
3130                 /*
3131                  * The HW reports non-shareable, we must remove the
3132                  * cacheability attributes as well.
3133                  */
3134                 val &= ~(GICR_PENDBASER_SHAREABILITY_MASK |
3135                          GICR_PENDBASER_CACHEABILITY_MASK);
3136                 val |= GICR_PENDBASER_nC;
3137                 gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
3138         }
3139
3140         /* Enable LPIs */
3141         val = readl_relaxed(rbase + GICR_CTLR);
3142         val |= GICR_CTLR_ENABLE_LPIS;
3143         writel_relaxed(val, rbase + GICR_CTLR);
3144
3145         if (gic_rdists->has_vlpis && !gic_rdists->has_rvpeid) {
3146                 void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3147
3148                 /*
3149                  * It's possible for CPU to receive VLPIs before it is
3150                  * scheduled as a vPE, especially for the first CPU, and the
3151                  * VLPI with INTID larger than 2^(IDbits+1) will be considered
3152                  * as out of range and dropped by GIC.
3153                  * So we initialize IDbits to known value to avoid VLPI drop.
3154                  */
3155                 val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
3156                 pr_debug("GICv4: CPU%d: Init IDbits to 0x%llx for GICR_VPROPBASER\n",
3157                         smp_processor_id(), val);
3158                 gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
3159
3160                 /*
3161                  * Also clear Valid bit of GICR_VPENDBASER, in case some
3162                  * ancient programming gets left in and has possibility of
3163                  * corrupting memory.
3164                  */
3165                 val = its_clear_vpend_valid(vlpi_base, 0, 0);
3166         }
3167
3168         if (allocate_vpe_l1_table()) {
3169                 /*
3170                  * If the allocation has failed, we're in massive trouble.
3171                  * Disable direct injection, and pray that no VM was
3172                  * already running...
3173                  */
3174                 gic_rdists->has_rvpeid = false;
3175                 gic_rdists->has_vlpis = false;
3176         }
3177
3178         /* Make sure the GIC has seen the above */
3179         dsb(sy);
3180 out:
3181         gic_data_rdist()->lpi_enabled = true;
3182         pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n",
3183                 smp_processor_id(),
3184                 gic_data_rdist()->pend_page ? "allocated" : "reserved",
3185                 &paddr);
3186 }
3187
3188 static void its_cpu_init_collection(struct its_node *its)
3189 {
3190         int cpu = smp_processor_id();
3191         u64 target;
3192
3193         /* avoid cross node collections and its mapping */
3194         if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
3195                 struct device_node *cpu_node;
3196
3197                 cpu_node = of_get_cpu_node(cpu, NULL);
3198                 if (its->numa_node != NUMA_NO_NODE &&
3199                         its->numa_node != of_node_to_nid(cpu_node))
3200                         return;
3201         }
3202
3203         /*
3204          * We now have to bind each collection to its target
3205          * redistributor.
3206          */
3207         if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) {
3208                 /*
3209                  * This ITS wants the physical address of the
3210                  * redistributor.
3211                  */
3212                 target = gic_data_rdist()->phys_base;
3213         } else {
3214                 /* This ITS wants a linear CPU number. */
3215                 target = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
3216                 target = GICR_TYPER_CPU_NUMBER(target) << 16;
3217         }
3218
3219         /* Perform collection mapping */
3220         its->collections[cpu].target_address = target;
3221         its->collections[cpu].col_id = cpu;
3222
3223         its_send_mapc(its, &its->collections[cpu], 1);
3224         its_send_invall(its, &its->collections[cpu]);
3225 }
3226
3227 static void its_cpu_init_collections(void)
3228 {
3229         struct its_node *its;
3230
3231         raw_spin_lock(&its_lock);
3232
3233         list_for_each_entry(its, &its_nodes, entry)
3234                 its_cpu_init_collection(its);
3235
3236         raw_spin_unlock(&its_lock);
3237 }
3238
3239 static struct its_device *its_find_device(struct its_node *its, u32 dev_id)
3240 {
3241         struct its_device *its_dev = NULL, *tmp;
3242         unsigned long flags;
3243
3244         raw_spin_lock_irqsave(&its->lock, flags);
3245
3246         list_for_each_entry(tmp, &its->its_device_list, entry) {
3247                 if (tmp->device_id == dev_id) {
3248                         its_dev = tmp;
3249                         break;
3250                 }
3251         }
3252
3253         raw_spin_unlock_irqrestore(&its->lock, flags);
3254
3255         return its_dev;
3256 }
3257
3258 static struct its_baser *its_get_baser(struct its_node *its, u32 type)
3259 {
3260         int i;
3261
3262         for (i = 0; i < GITS_BASER_NR_REGS; i++) {
3263                 if (GITS_BASER_TYPE(its->tables[i].val) == type)
3264                         return &its->tables[i];
3265         }
3266
3267         return NULL;
3268 }
3269
3270 static bool its_alloc_table_entry(struct its_node *its,
3271                                   struct its_baser *baser, u32 id)
3272 {
3273         struct page *page;
3274         u32 esz, idx;
3275         __le64 *table;
3276
3277         /* Don't allow device id that exceeds single, flat table limit */
3278         esz = GITS_BASER_ENTRY_SIZE(baser->val);
3279         if (!(baser->val & GITS_BASER_INDIRECT))
3280                 return (id < (PAGE_ORDER_TO_SIZE(baser->order) / esz));
3281
3282         /* Compute 1st level table index & check if that exceeds table limit */
3283         idx = id >> ilog2(baser->psz / esz);
3284         if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE))
3285                 return false;
3286
3287         table = baser->base;
3288
3289         /* Allocate memory for 2nd level table */
3290         if (!table[idx]) {
3291                 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
3292                                         get_order(baser->psz));
3293                 if (!page)
3294                         return false;
3295
3296                 /* Flush Lvl2 table to PoC if hw doesn't support coherency */
3297                 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
3298                         gic_flush_dcache_to_poc(page_address(page), baser->psz);
3299
3300                 table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
3301
3302                 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
3303                 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
3304                         gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
3305
3306                 /* Ensure updated table contents are visible to ITS hardware */
3307                 dsb(sy);
3308         }
3309
3310         return true;
3311 }
3312
3313 static bool its_alloc_device_table(struct its_node *its, u32 dev_id)
3314 {
3315         struct its_baser *baser;
3316
3317         baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE);
3318
3319         /* Don't allow device id that exceeds ITS hardware limit */
3320         if (!baser)
3321                 return (ilog2(dev_id) < device_ids(its));
3322
3323         return its_alloc_table_entry(its, baser, dev_id);
3324 }
3325
3326 static bool its_alloc_vpe_table(u32 vpe_id)
3327 {
3328         struct its_node *its;
3329         int cpu;
3330
3331         /*
3332          * Make sure the L2 tables are allocated on *all* v4 ITSs. We
3333          * could try and only do it on ITSs corresponding to devices
3334          * that have interrupts targeted at this VPE, but the
3335          * complexity becomes crazy (and you have tons of memory
3336          * anyway, right?).
3337          */
3338         list_for_each_entry(its, &its_nodes, entry) {
3339                 struct its_baser *baser;
3340
3341                 if (!is_v4(its))
3342                         continue;
3343
3344                 baser = its_get_baser(its, GITS_BASER_TYPE_VCPU);
3345                 if (!baser)
3346                         return false;
3347
3348                 if (!its_alloc_table_entry(its, baser, vpe_id))
3349                         return false;
3350         }
3351
3352         /* Non v4.1? No need to iterate RDs and go back early. */
3353         if (!gic_rdists->has_rvpeid)
3354                 return true;
3355
3356         /*
3357          * Make sure the L2 tables are allocated for all copies of
3358          * the L1 table on *all* v4.1 RDs.
3359          */
3360         for_each_possible_cpu(cpu) {
3361                 if (!allocate_vpe_l2_table(cpu, vpe_id))
3362                         return false;
3363         }
3364
3365         return true;
3366 }
3367
3368 static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
3369                                             int nvecs, bool alloc_lpis)
3370 {
3371         struct its_device *dev;
3372         unsigned long *lpi_map = NULL;
3373         unsigned long flags;
3374         u16 *col_map = NULL;
3375         void *itt;
3376         int lpi_base;
3377         int nr_lpis;
3378         int nr_ites;
3379         int sz;
3380
3381         if (!its_alloc_device_table(its, dev_id))
3382                 return NULL;
3383
3384         if (WARN_ON(!is_power_of_2(nvecs)))
3385                 nvecs = roundup_pow_of_two(nvecs);
3386
3387         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
3388         /*
3389          * Even if the device wants a single LPI, the ITT must be
3390          * sized as a power of two (and you need at least one bit...).
3391          */
3392         nr_ites = max(2, nvecs);
3393         sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1);
3394         sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1;
3395         itt = kzalloc_node(sz, GFP_KERNEL, its->numa_node);
3396         if (alloc_lpis) {
3397                 lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis);
3398                 if (lpi_map)
3399                         col_map = kcalloc(nr_lpis, sizeof(*col_map),
3400                                           GFP_KERNEL);
3401         } else {
3402                 col_map = kcalloc(nr_ites, sizeof(*col_map), GFP_KERNEL);
3403                 nr_lpis = 0;
3404                 lpi_base = 0;
3405         }
3406
3407         if (!dev || !itt ||  !col_map || (!lpi_map && alloc_lpis)) {
3408                 kfree(dev);
3409                 kfree(itt);
3410                 kfree(lpi_map);
3411                 kfree(col_map);
3412                 return NULL;
3413         }
3414
3415         gic_flush_dcache_to_poc(itt, sz);
3416
3417         dev->its = its;
3418         dev->itt = itt;
3419         dev->nr_ites = nr_ites;
3420         dev->event_map.lpi_map = lpi_map;
3421         dev->event_map.col_map = col_map;
3422         dev->event_map.lpi_base = lpi_base;
3423         dev->event_map.nr_lpis = nr_lpis;
3424         raw_spin_lock_init(&dev->event_map.vlpi_lock);
3425         dev->device_id = dev_id;
3426         INIT_LIST_HEAD(&dev->entry);
3427
3428         raw_spin_lock_irqsave(&its->lock, flags);
3429         list_add(&dev->entry, &its->its_device_list);
3430         raw_spin_unlock_irqrestore(&its->lock, flags);
3431
3432         /* Map device to its ITT */
3433         its_send_mapd(dev, 1);
3434
3435         return dev;
3436 }
3437
3438 static void its_free_device(struct its_device *its_dev)
3439 {
3440         unsigned long flags;
3441
3442         raw_spin_lock_irqsave(&its_dev->its->lock, flags);
3443         list_del(&its_dev->entry);
3444         raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);
3445         kfree(its_dev->event_map.col_map);
3446         kfree(its_dev->itt);
3447         kfree(its_dev);
3448 }
3449
3450 static int its_alloc_device_irq(struct its_device *dev, int nvecs, irq_hw_number_t *hwirq)
3451 {
3452         int idx;
3453
3454         /* Find a free LPI region in lpi_map and allocate them. */
3455         idx = bitmap_find_free_region(dev->event_map.lpi_map,
3456                                       dev->event_map.nr_lpis,
3457                                       get_count_order(nvecs));
3458         if (idx < 0)
3459                 return -ENOSPC;
3460
3461         *hwirq = dev->event_map.lpi_base + idx;
3462
3463         return 0;
3464 }
3465
3466 static int its_msi_prepare(struct irq_domain *domain, struct device *dev,
3467                            int nvec, msi_alloc_info_t *info)
3468 {
3469         struct its_node *its;
3470         struct its_device *its_dev;
3471         struct msi_domain_info *msi_info;
3472         u32 dev_id;
3473         int err = 0;
3474
3475         /*
3476          * We ignore "dev" entirely, and rely on the dev_id that has
3477          * been passed via the scratchpad. This limits this domain's
3478          * usefulness to upper layers that definitely know that they
3479          * are built on top of the ITS.
3480          */
3481         dev_id = info->scratchpad[0].ul;
3482
3483         msi_info = msi_get_domain_info(domain);
3484         its = msi_info->data;
3485
3486         if (!gic_rdists->has_direct_lpi &&
3487             vpe_proxy.dev &&
3488             vpe_proxy.dev->its == its &&
3489             dev_id == vpe_proxy.dev->device_id) {
3490                 /* Bad luck. Get yourself a better implementation */
3491                 WARN_ONCE(1, "DevId %x clashes with GICv4 VPE proxy device\n",
3492                           dev_id);
3493                 return -EINVAL;
3494         }
3495
3496         mutex_lock(&its->dev_alloc_lock);
3497         its_dev = its_find_device(its, dev_id);
3498         if (its_dev) {
3499                 /*
3500                  * We already have seen this ID, probably through
3501                  * another alias (PCI bridge of some sort). No need to
3502                  * create the device.
3503                  */
3504                 its_dev->shared = true;
3505                 pr_debug("Reusing ITT for devID %x\n", dev_id);
3506                 goto out;
3507         }
3508
3509         its_dev = its_create_device(its, dev_id, nvec, true);
3510         if (!its_dev) {
3511                 err = -ENOMEM;
3512                 goto out;
3513         }
3514
3515         pr_debug("ITT %d entries, %d bits\n", nvec, ilog2(nvec));
3516 out:
3517         mutex_unlock(&its->dev_alloc_lock);
3518         info->scratchpad[0].ptr = its_dev;
3519         return err;
3520 }
3521
3522 static struct msi_domain_ops its_msi_domain_ops = {
3523         .msi_prepare    = its_msi_prepare,
3524 };
3525
3526 static int its_irq_gic_domain_alloc(struct irq_domain *domain,
3527                                     unsigned int virq,
3528                                     irq_hw_number_t hwirq)
3529 {
3530         struct irq_fwspec fwspec;
3531
3532         if (irq_domain_get_of_node(domain->parent)) {
3533                 fwspec.fwnode = domain->parent->fwnode;
3534                 fwspec.param_count = 3;
3535                 fwspec.param[0] = GIC_IRQ_TYPE_LPI;
3536                 fwspec.param[1] = hwirq;
3537                 fwspec.param[2] = IRQ_TYPE_EDGE_RISING;
3538         } else if (is_fwnode_irqchip(domain->parent->fwnode)) {
3539                 fwspec.fwnode = domain->parent->fwnode;
3540                 fwspec.param_count = 2;
3541                 fwspec.param[0] = hwirq;
3542                 fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
3543         } else {
3544                 return -EINVAL;
3545         }
3546
3547         return irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec);
3548 }
3549
3550 static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
3551                                 unsigned int nr_irqs, void *args)
3552 {
3553         msi_alloc_info_t *info = args;
3554         struct its_device *its_dev = info->scratchpad[0].ptr;
3555         struct its_node *its = its_dev->its;
3556         struct irq_data *irqd;
3557         irq_hw_number_t hwirq;
3558         int err;
3559         int i;
3560
3561         err = its_alloc_device_irq(its_dev, nr_irqs, &hwirq);
3562         if (err)
3563                 return err;
3564
3565         err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev));
3566         if (err)
3567                 return err;
3568
3569         for (i = 0; i < nr_irqs; i++) {
3570                 err = its_irq_gic_domain_alloc(domain, virq + i, hwirq + i);
3571                 if (err)
3572                         return err;
3573
3574                 irq_domain_set_hwirq_and_chip(domain, virq + i,
3575                                               hwirq + i, &its_irq_chip, its_dev);
3576                 irqd = irq_get_irq_data(virq + i);
3577                 irqd_set_single_target(irqd);
3578                 irqd_set_affinity_on_activate(irqd);
3579                 pr_debug("ID:%d pID:%d vID:%d\n",
3580                          (int)(hwirq + i - its_dev->event_map.lpi_base),
3581                          (int)(hwirq + i), virq + i);
3582         }
3583
3584         return 0;
3585 }
3586
3587 static int its_irq_domain_activate(struct irq_domain *domain,
3588                                    struct irq_data *d, bool reserve)
3589 {
3590         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3591         u32 event = its_get_event_id(d);
3592         int cpu;
3593
3594         cpu = its_select_cpu(d, cpu_online_mask);
3595         if (cpu < 0 || cpu >= nr_cpu_ids)
3596                 return -EINVAL;
3597
3598         its_inc_lpi_count(d, cpu);
3599         its_dev->event_map.col_map[event] = cpu;
3600         irq_data_update_effective_affinity(d, cpumask_of(cpu));
3601
3602         /* Map the GIC IRQ and event to the device */
3603         its_send_mapti(its_dev, d->hwirq, event);
3604         return 0;
3605 }
3606
3607 static void its_irq_domain_deactivate(struct irq_domain *domain,
3608                                       struct irq_data *d)
3609 {
3610         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3611         u32 event = its_get_event_id(d);
3612
3613         its_dec_lpi_count(d, its_dev->event_map.col_map[event]);
3614         /* Stop the delivery of interrupts */
3615         its_send_discard(its_dev, event);
3616 }
3617
3618 static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq,
3619                                 unsigned int nr_irqs)
3620 {
3621         struct irq_data *d = irq_domain_get_irq_data(domain, virq);
3622         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3623         struct its_node *its = its_dev->its;
3624         int i;
3625
3626         bitmap_release_region(its_dev->event_map.lpi_map,
3627                               its_get_event_id(irq_domain_get_irq_data(domain, virq)),
3628                               get_count_order(nr_irqs));
3629
3630         for (i = 0; i < nr_irqs; i++) {
3631                 struct irq_data *data = irq_domain_get_irq_data(domain,
3632                                                                 virq + i);
3633                 /* Nuke the entry in the domain */
3634                 irq_domain_reset_irq_data(data);
3635         }
3636
3637         mutex_lock(&its->dev_alloc_lock);
3638
3639         /*
3640          * If all interrupts have been freed, start mopping the
3641          * floor. This is conditioned on the device not being shared.
3642          */
3643         if (!its_dev->shared &&
3644             bitmap_empty(its_dev->event_map.lpi_map,
3645                          its_dev->event_map.nr_lpis)) {
3646                 its_lpi_free(its_dev->event_map.lpi_map,
3647                              its_dev->event_map.lpi_base,
3648                              its_dev->event_map.nr_lpis);
3649
3650                 /* Unmap device/itt */
3651                 its_send_mapd(its_dev, 0);
3652                 its_free_device(its_dev);
3653         }
3654
3655         mutex_unlock(&its->dev_alloc_lock);
3656
3657         irq_domain_free_irqs_parent(domain, virq, nr_irqs);
3658 }
3659
3660 static const struct irq_domain_ops its_domain_ops = {
3661         .alloc                  = its_irq_domain_alloc,
3662         .free                   = its_irq_domain_free,
3663         .activate               = its_irq_domain_activate,
3664         .deactivate             = its_irq_domain_deactivate,
3665 };
3666
3667 /*
3668  * This is insane.
3669  *
3670  * If a GICv4.0 doesn't implement Direct LPIs (which is extremely
3671  * likely), the only way to perform an invalidate is to use a fake
3672  * device to issue an INV command, implying that the LPI has first
3673  * been mapped to some event on that device. Since this is not exactly
3674  * cheap, we try to keep that mapping around as long as possible, and
3675  * only issue an UNMAP if we're short on available slots.
3676  *
3677  * Broken by design(tm).
3678  *
3679  * GICv4.1, on the other hand, mandates that we're able to invalidate
3680  * by writing to a MMIO register. It doesn't implement the whole of
3681  * DirectLPI, but that's good enough. And most of the time, we don't
3682  * even have to invalidate anything, as the redistributor can be told
3683  * whether to generate a doorbell or not (we thus leave it enabled,
3684  * always).
3685  */
3686 static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe)
3687 {
3688         /* GICv4.1 doesn't use a proxy, so nothing to do here */
3689         if (gic_rdists->has_rvpeid)
3690                 return;
3691
3692         /* Already unmapped? */
3693         if (vpe->vpe_proxy_event == -1)
3694                 return;
3695
3696         its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event);
3697         vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL;
3698
3699         /*
3700          * We don't track empty slots at all, so let's move the
3701          * next_victim pointer if we can quickly reuse that slot
3702          * instead of nuking an existing entry. Not clear that this is
3703          * always a win though, and this might just generate a ripple
3704          * effect... Let's just hope VPEs don't migrate too often.
3705          */
3706         if (vpe_proxy.vpes[vpe_proxy.next_victim])
3707                 vpe_proxy.next_victim = vpe->vpe_proxy_event;
3708
3709         vpe->vpe_proxy_event = -1;
3710 }
3711
3712 static void its_vpe_db_proxy_unmap(struct its_vpe *vpe)
3713 {
3714         /* GICv4.1 doesn't use a proxy, so nothing to do here */
3715         if (gic_rdists->has_rvpeid)
3716                 return;
3717
3718         if (!gic_rdists->has_direct_lpi) {
3719                 unsigned long flags;
3720
3721                 raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3722                 its_vpe_db_proxy_unmap_locked(vpe);
3723                 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3724         }
3725 }
3726
3727 static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe)
3728 {
3729         /* GICv4.1 doesn't use a proxy, so nothing to do here */
3730         if (gic_rdists->has_rvpeid)
3731                 return;
3732
3733         /* Already mapped? */
3734         if (vpe->vpe_proxy_event != -1)
3735                 return;
3736
3737         /* This slot was already allocated. Kick the other VPE out. */
3738         if (vpe_proxy.vpes[vpe_proxy.next_victim])
3739                 its_vpe_db_proxy_unmap_locked(vpe_proxy.vpes[vpe_proxy.next_victim]);
3740
3741         /* Map the new VPE instead */
3742         vpe_proxy.vpes[vpe_proxy.next_victim] = vpe;
3743         vpe->vpe_proxy_event = vpe_proxy.next_victim;
3744         vpe_proxy.next_victim = (vpe_proxy.next_victim + 1) % vpe_proxy.dev->nr_ites;
3745
3746         vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx;
3747         its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event);
3748 }
3749
3750 static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to)
3751 {
3752         unsigned long flags;
3753         struct its_collection *target_col;
3754
3755         /* GICv4.1 doesn't use a proxy, so nothing to do here */
3756         if (gic_rdists->has_rvpeid)
3757                 return;
3758
3759         if (gic_rdists->has_direct_lpi) {
3760                 void __iomem *rdbase;
3761
3762                 rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base;
3763                 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
3764                 wait_for_syncr(rdbase);
3765
3766                 return;
3767         }
3768
3769         raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3770
3771         its_vpe_db_proxy_map_locked(vpe);
3772
3773         target_col = &vpe_proxy.dev->its->collections[to];
3774         its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event);
3775         vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to;
3776
3777         raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3778 }
3779
3780 static int its_vpe_set_affinity(struct irq_data *d,
3781                                 const struct cpumask *mask_val,
3782                                 bool force)
3783 {
3784         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3785         struct cpumask common, *table_mask;
3786         unsigned long flags;
3787         int from, cpu;
3788
3789         /*
3790          * Changing affinity is mega expensive, so let's be as lazy as
3791          * we can and only do it if we really have to. Also, if mapped
3792          * into the proxy device, we need to move the doorbell
3793          * interrupt to its new location.
3794          *
3795          * Another thing is that changing the affinity of a vPE affects
3796          * *other interrupts* such as all the vLPIs that are routed to
3797          * this vPE. This means that the irq_desc lock is not enough to
3798          * protect us, and that we must ensure nobody samples vpe->col_idx
3799          * during the update, hence the lock below which must also be
3800          * taken on any vLPI handling path that evaluates vpe->col_idx.
3801          */
3802         from = vpe_to_cpuid_lock(vpe, &flags);
3803         table_mask = gic_data_rdist_cpu(from)->vpe_table_mask;
3804
3805         /*
3806          * If we are offered another CPU in the same GICv4.1 ITS
3807          * affinity, pick this one. Otherwise, any CPU will do.
3808          */
3809         if (table_mask && cpumask_and(&common, mask_val, table_mask))
3810                 cpu = cpumask_test_cpu(from, &common) ? from : cpumask_first(&common);
3811         else
3812                 cpu = cpumask_first(mask_val);
3813
3814         if (from == cpu)
3815                 goto out;
3816
3817         vpe->col_idx = cpu;
3818
3819         its_send_vmovp(vpe);
3820         its_vpe_db_proxy_move(vpe, from, cpu);
3821
3822 out:
3823         irq_data_update_effective_affinity(d, cpumask_of(cpu));
3824         vpe_to_cpuid_unlock(vpe, flags);
3825
3826         return IRQ_SET_MASK_OK_DONE;
3827 }
3828
3829 static void its_wait_vpt_parse_complete(void)
3830 {
3831         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3832         u64 val;
3833
3834         if (!gic_rdists->has_vpend_valid_dirty)
3835                 return;
3836
3837         WARN_ON_ONCE(readq_relaxed_poll_timeout_atomic(vlpi_base + GICR_VPENDBASER,
3838                                                        val,
3839                                                        !(val & GICR_VPENDBASER_Dirty),
3840                                                        10, 500));
3841 }
3842
3843 static void its_vpe_schedule(struct its_vpe *vpe)
3844 {
3845         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3846         u64 val;
3847
3848         /* Schedule the VPE */
3849         val  = virt_to_phys(page_address(vpe->its_vm->vprop_page)) &
3850                 GENMASK_ULL(51, 12);
3851         val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
3852         val |= GICR_VPROPBASER_RaWb;
3853         val |= GICR_VPROPBASER_InnerShareable;
3854         gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
3855
3856         val  = virt_to_phys(page_address(vpe->vpt_page)) &
3857                 GENMASK_ULL(51, 16);
3858         val |= GICR_VPENDBASER_RaWaWb;
3859         val |= GICR_VPENDBASER_InnerShareable;
3860         /*
3861          * There is no good way of finding out if the pending table is
3862          * empty as we can race against the doorbell interrupt very
3863          * easily. So in the end, vpe->pending_last is only an
3864          * indication that the vcpu has something pending, not one
3865          * that the pending table is empty. A good implementation
3866          * would be able to read its coarse map pretty quickly anyway,
3867          * making this a tolerable issue.
3868          */
3869         val |= GICR_VPENDBASER_PendingLast;
3870         val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0;
3871         val |= GICR_VPENDBASER_Valid;
3872         gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
3873
3874         its_wait_vpt_parse_complete();
3875 }
3876
3877 static void its_vpe_deschedule(struct its_vpe *vpe)
3878 {
3879         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3880         u64 val;
3881
3882         val = its_clear_vpend_valid(vlpi_base, 0, 0);
3883
3884         vpe->idai = !!(val & GICR_VPENDBASER_IDAI);
3885         vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
3886 }
3887
3888 static void its_vpe_invall(struct its_vpe *vpe)
3889 {
3890         struct its_node *its;
3891
3892         list_for_each_entry(its, &its_nodes, entry) {
3893                 if (!is_v4(its))
3894                         continue;
3895
3896                 if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr])
3897                         continue;
3898
3899                 /*
3900                  * Sending a VINVALL to a single ITS is enough, as all
3901                  * we need is to reach the redistributors.
3902                  */
3903                 its_send_vinvall(its, vpe);
3904                 return;
3905         }
3906 }
3907
3908 static int its_vpe_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
3909 {
3910         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3911         struct its_cmd_info *info = vcpu_info;
3912
3913         switch (info->cmd_type) {
3914         case SCHEDULE_VPE:
3915                 its_vpe_schedule(vpe);
3916                 return 0;
3917
3918         case DESCHEDULE_VPE:
3919                 its_vpe_deschedule(vpe);
3920                 return 0;
3921
3922         case INVALL_VPE:
3923                 its_vpe_invall(vpe);
3924                 return 0;
3925
3926         default:
3927                 return -EINVAL;
3928         }
3929 }
3930
3931 static void its_vpe_send_cmd(struct its_vpe *vpe,
3932                              void (*cmd)(struct its_device *, u32))
3933 {
3934         unsigned long flags;
3935
3936         raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3937
3938         its_vpe_db_proxy_map_locked(vpe);
3939         cmd(vpe_proxy.dev, vpe->vpe_proxy_event);
3940
3941         raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3942 }
3943
3944 static void its_vpe_send_inv(struct irq_data *d)
3945 {
3946         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3947
3948         if (gic_rdists->has_direct_lpi)
3949                 __direct_lpi_inv(d, d->parent_data->hwirq);
3950         else
3951                 its_vpe_send_cmd(vpe, its_send_inv);
3952 }
3953
3954 static void its_vpe_mask_irq(struct irq_data *d)
3955 {
3956         /*
3957          * We need to unmask the LPI, which is described by the parent
3958          * irq_data. Instead of calling into the parent (which won't
3959          * exactly do the right thing, let's simply use the
3960          * parent_data pointer. Yes, I'm naughty.
3961          */
3962         lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
3963         its_vpe_send_inv(d);
3964 }
3965
3966 static void its_vpe_unmask_irq(struct irq_data *d)
3967 {
3968         /* Same hack as above... */
3969         lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
3970         its_vpe_send_inv(d);
3971 }
3972
3973 static int its_vpe_set_irqchip_state(struct irq_data *d,
3974                                      enum irqchip_irq_state which,
3975                                      bool state)
3976 {
3977         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3978
3979         if (which != IRQCHIP_STATE_PENDING)
3980                 return -EINVAL;
3981
3982         if (gic_rdists->has_direct_lpi) {
3983                 void __iomem *rdbase;
3984
3985                 rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
3986                 if (state) {
3987                         gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR);
3988                 } else {
3989                         gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
3990                         wait_for_syncr(rdbase);
3991                 }
3992         } else {
3993                 if (state)
3994                         its_vpe_send_cmd(vpe, its_send_int);
3995                 else
3996                         its_vpe_send_cmd(vpe, its_send_clear);
3997         }
3998
3999         return 0;
4000 }
4001
4002 static int its_vpe_retrigger(struct irq_data *d)
4003 {
4004         return !its_vpe_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
4005 }
4006
4007 static struct irq_chip its_vpe_irq_chip = {
4008         .name                   = "GICv4-vpe",
4009         .irq_mask               = its_vpe_mask_irq,
4010         .irq_unmask             = its_vpe_unmask_irq,
4011         .irq_eoi                = irq_chip_eoi_parent,
4012         .irq_set_affinity       = its_vpe_set_affinity,
4013         .irq_retrigger          = its_vpe_retrigger,
4014         .irq_set_irqchip_state  = its_vpe_set_irqchip_state,
4015         .irq_set_vcpu_affinity  = its_vpe_set_vcpu_affinity,
4016 };
4017
4018 static struct its_node *find_4_1_its(void)
4019 {
4020         static struct its_node *its = NULL;
4021
4022         if (!its) {
4023                 list_for_each_entry(its, &its_nodes, entry) {
4024                         if (is_v4_1(its))
4025                                 return its;
4026                 }
4027
4028                 /* Oops? */
4029                 its = NULL;
4030         }
4031
4032         return its;
4033 }
4034
4035 static void its_vpe_4_1_send_inv(struct irq_data *d)
4036 {
4037         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4038         struct its_node *its;
4039
4040         /*
4041          * GICv4.1 wants doorbells to be invalidated using the
4042          * INVDB command in order to be broadcast to all RDs. Send
4043          * it to the first valid ITS, and let the HW do its magic.
4044          */
4045         its = find_4_1_its();
4046         if (its)
4047                 its_send_invdb(its, vpe);
4048 }
4049
4050 static void its_vpe_4_1_mask_irq(struct irq_data *d)
4051 {
4052         lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
4053         its_vpe_4_1_send_inv(d);
4054 }
4055
4056 static void its_vpe_4_1_unmask_irq(struct irq_data *d)
4057 {
4058         lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
4059         its_vpe_4_1_send_inv(d);
4060 }
4061
4062 static void its_vpe_4_1_schedule(struct its_vpe *vpe,
4063                                  struct its_cmd_info *info)
4064 {
4065         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
4066         u64 val = 0;
4067
4068         /* Schedule the VPE */
4069         val |= GICR_VPENDBASER_Valid;
4070         val |= info->g0en ? GICR_VPENDBASER_4_1_VGRP0EN : 0;
4071         val |= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0;
4072         val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id);
4073
4074         gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
4075
4076         its_wait_vpt_parse_complete();
4077 }
4078
4079 static void its_vpe_4_1_deschedule(struct its_vpe *vpe,
4080                                    struct its_cmd_info *info)
4081 {
4082         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
4083         u64 val;
4084
4085         if (info->req_db) {
4086                 unsigned long flags;
4087
4088                 /*
4089                  * vPE is going to block: make the vPE non-resident with
4090                  * PendingLast clear and DB set. The GIC guarantees that if
4091                  * we read-back PendingLast clear, then a doorbell will be
4092                  * delivered when an interrupt comes.
4093                  *
4094                  * Note the locking to deal with the concurrent update of
4095                  * pending_last from the doorbell interrupt handler that can
4096                  * run concurrently.
4097                  */
4098                 raw_spin_lock_irqsave(&vpe->vpe_lock, flags);
4099                 val = its_clear_vpend_valid(vlpi_base,
4100                                             GICR_VPENDBASER_PendingLast,
4101                                             GICR_VPENDBASER_4_1_DB);
4102                 vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
4103                 raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
4104         } else {
4105                 /*
4106                  * We're not blocking, so just make the vPE non-resident
4107                  * with PendingLast set, indicating that we'll be back.
4108                  */
4109                 val = its_clear_vpend_valid(vlpi_base,
4110                                             0,
4111                                             GICR_VPENDBASER_PendingLast);
4112                 vpe->pending_last = true;
4113         }
4114 }
4115
4116 static void its_vpe_4_1_invall(struct its_vpe *vpe)
4117 {
4118         void __iomem *rdbase;
4119         unsigned long flags;
4120         u64 val;
4121         int cpu;
4122
4123         val  = GICR_INVALLR_V;
4124         val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id);
4125
4126         /* Target the redistributor this vPE is currently known on */
4127         cpu = vpe_to_cpuid_lock(vpe, &flags);
4128         raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
4129         rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
4130         gic_write_lpir(val, rdbase + GICR_INVALLR);
4131
4132         wait_for_syncr(rdbase);
4133         raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
4134         vpe_to_cpuid_unlock(vpe, flags);
4135 }
4136
4137 static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
4138 {
4139         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4140         struct its_cmd_info *info = vcpu_info;
4141
4142         switch (info->cmd_type) {
4143         case SCHEDULE_VPE:
4144                 its_vpe_4_1_schedule(vpe, info);
4145                 return 0;
4146
4147         case DESCHEDULE_VPE:
4148                 its_vpe_4_1_deschedule(vpe, info);
4149                 return 0;
4150
4151         case INVALL_VPE:
4152                 its_vpe_4_1_invall(vpe);
4153                 return 0;
4154
4155         default:
4156                 return -EINVAL;
4157         }
4158 }
4159
4160 static struct irq_chip its_vpe_4_1_irq_chip = {
4161         .name                   = "GICv4.1-vpe",
4162         .irq_mask               = its_vpe_4_1_mask_irq,
4163         .irq_unmask             = its_vpe_4_1_unmask_irq,
4164         .irq_eoi                = irq_chip_eoi_parent,
4165         .irq_set_affinity       = its_vpe_set_affinity,
4166         .irq_set_vcpu_affinity  = its_vpe_4_1_set_vcpu_affinity,
4167 };
4168
4169 static void its_configure_sgi(struct irq_data *d, bool clear)
4170 {
4171         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4172         struct its_cmd_desc desc;
4173
4174         desc.its_vsgi_cmd.vpe = vpe;
4175         desc.its_vsgi_cmd.sgi = d->hwirq;
4176         desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority;
4177         desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled;
4178         desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group;
4179         desc.its_vsgi_cmd.clear = clear;
4180
4181         /*
4182          * GICv4.1 allows us to send VSGI commands to any ITS as long as the
4183          * destination VPE is mapped there. Since we map them eagerly at
4184          * activation time, we're pretty sure the first GICv4.1 ITS will do.
4185          */
4186         its_send_single_vcommand(find_4_1_its(), its_build_vsgi_cmd, &desc);
4187 }
4188
4189 static void its_sgi_mask_irq(struct irq_data *d)
4190 {
4191         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4192
4193         vpe->sgi_config[d->hwirq].enabled = false;
4194         its_configure_sgi(d, false);
4195 }
4196
4197 static void its_sgi_unmask_irq(struct irq_data *d)
4198 {
4199         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4200
4201         vpe->sgi_config[d->hwirq].enabled = true;
4202         its_configure_sgi(d, false);
4203 }
4204
4205 static int its_sgi_set_affinity(struct irq_data *d,
4206                                 const struct cpumask *mask_val,
4207                                 bool force)
4208 {
4209         /*
4210          * There is no notion of affinity for virtual SGIs, at least
4211          * not on the host (since they can only be targeting a vPE).
4212          * Tell the kernel we've done whatever it asked for.
4213          */
4214         irq_data_update_effective_affinity(d, mask_val);
4215         return IRQ_SET_MASK_OK;
4216 }
4217
4218 static int its_sgi_set_irqchip_state(struct irq_data *d,
4219                                      enum irqchip_irq_state which,
4220                                      bool state)
4221 {
4222         if (which != IRQCHIP_STATE_PENDING)
4223                 return -EINVAL;
4224
4225         if (state) {
4226                 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4227                 struct its_node *its = find_4_1_its();
4228                 u64 val;
4229
4230                 val  = FIELD_PREP(GITS_SGIR_VPEID, vpe->vpe_id);
4231                 val |= FIELD_PREP(GITS_SGIR_VINTID, d->hwirq);
4232                 writeq_relaxed(val, its->sgir_base + GITS_SGIR - SZ_128K);
4233         } else {
4234                 its_configure_sgi(d, true);
4235         }
4236
4237         return 0;
4238 }
4239
4240 static int its_sgi_get_irqchip_state(struct irq_data *d,
4241                                      enum irqchip_irq_state which, bool *val)
4242 {
4243         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4244         void __iomem *base;
4245         unsigned long flags;
4246         u32 count = 1000000;    /* 1s! */
4247         u32 status;
4248         int cpu;
4249
4250         if (which != IRQCHIP_STATE_PENDING)
4251                 return -EINVAL;
4252
4253         /*
4254          * Locking galore! We can race against two different events:
4255          *
4256          * - Concurrent vPE affinity change: we must make sure it cannot
4257          *   happen, or we'll talk to the wrong redistributor. This is
4258          *   identical to what happens with vLPIs.
4259          *
4260          * - Concurrent VSGIPENDR access: As it involves accessing two
4261          *   MMIO registers, this must be made atomic one way or another.
4262          */
4263         cpu = vpe_to_cpuid_lock(vpe, &flags);
4264         raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
4265         base = gic_data_rdist_cpu(cpu)->rd_base + SZ_128K;
4266         writel_relaxed(vpe->vpe_id, base + GICR_VSGIR);
4267         do {
4268                 status = readl_relaxed(base + GICR_VSGIPENDR);
4269                 if (!(status & GICR_VSGIPENDR_BUSY))
4270                         goto out;
4271
4272                 count--;
4273                 if (!count) {
4274                         pr_err_ratelimited("Unable to get SGI status\n");
4275                         goto out;
4276                 }
4277                 cpu_relax();
4278                 udelay(1);
4279         } while (count);
4280
4281 out:
4282         raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
4283         vpe_to_cpuid_unlock(vpe, flags);
4284
4285         if (!count)
4286                 return -ENXIO;
4287
4288         *val = !!(status & (1 << d->hwirq));
4289
4290         return 0;
4291 }
4292
4293 static int its_sgi_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
4294 {
4295         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4296         struct its_cmd_info *info = vcpu_info;
4297
4298         switch (info->cmd_type) {
4299         case PROP_UPDATE_VSGI:
4300                 vpe->sgi_config[d->hwirq].priority = info->priority;
4301                 vpe->sgi_config[d->hwirq].group = info->group;
4302                 its_configure_sgi(d, false);
4303                 return 0;
4304
4305         default:
4306                 return -EINVAL;
4307         }
4308 }
4309
4310 static struct irq_chip its_sgi_irq_chip = {
4311         .name                   = "GICv4.1-sgi",
4312         .irq_mask               = its_sgi_mask_irq,
4313         .irq_unmask             = its_sgi_unmask_irq,
4314         .irq_set_affinity       = its_sgi_set_affinity,
4315         .irq_set_irqchip_state  = its_sgi_set_irqchip_state,
4316         .irq_get_irqchip_state  = its_sgi_get_irqchip_state,
4317         .irq_set_vcpu_affinity  = its_sgi_set_vcpu_affinity,
4318 };
4319
4320 static int its_sgi_irq_domain_alloc(struct irq_domain *domain,
4321                                     unsigned int virq, unsigned int nr_irqs,
4322                                     void *args)
4323 {
4324         struct its_vpe *vpe = args;
4325         int i;
4326
4327         /* Yes, we do want 16 SGIs */
4328         WARN_ON(nr_irqs != 16);
4329
4330         for (i = 0; i < 16; i++) {
4331                 vpe->sgi_config[i].priority = 0;
4332                 vpe->sgi_config[i].enabled = false;
4333                 vpe->sgi_config[i].group = false;
4334
4335                 irq_domain_set_hwirq_and_chip(domain, virq + i, i,
4336                                               &its_sgi_irq_chip, vpe);
4337                 irq_set_status_flags(virq + i, IRQ_DISABLE_UNLAZY);
4338         }
4339
4340         return 0;
4341 }
4342
4343 static void its_sgi_irq_domain_free(struct irq_domain *domain,
4344                                     unsigned int virq,
4345                                     unsigned int nr_irqs)
4346 {
4347         /* Nothing to do */
4348 }
4349
4350 static int its_sgi_irq_domain_activate(struct irq_domain *domain,
4351                                        struct irq_data *d, bool reserve)
4352 {
4353         /* Write out the initial SGI configuration */
4354         its_configure_sgi(d, false);
4355         return 0;
4356 }
4357
4358 static void its_sgi_irq_domain_deactivate(struct irq_domain *domain,
4359                                           struct irq_data *d)
4360 {
4361         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4362
4363         /*
4364          * The VSGI command is awkward:
4365          *
4366          * - To change the configuration, CLEAR must be set to false,
4367          *   leaving the pending bit unchanged.
4368          * - To clear the pending bit, CLEAR must be set to true, leaving
4369          *   the configuration unchanged.
4370          *
4371          * You just can't do both at once, hence the two commands below.
4372          */
4373         vpe->sgi_config[d->hwirq].enabled = false;
4374         its_configure_sgi(d, false);
4375         its_configure_sgi(d, true);
4376 }
4377
4378 static const struct irq_domain_ops its_sgi_domain_ops = {
4379         .alloc          = its_sgi_irq_domain_alloc,
4380         .free           = its_sgi_irq_domain_free,
4381         .activate       = its_sgi_irq_domain_activate,
4382         .deactivate     = its_sgi_irq_domain_deactivate,
4383 };
4384
4385 static int its_vpe_id_alloc(void)
4386 {
4387         return ida_simple_get(&its_vpeid_ida, 0, ITS_MAX_VPEID, GFP_KERNEL);
4388 }
4389
4390 static void its_vpe_id_free(u16 id)
4391 {
4392         ida_simple_remove(&its_vpeid_ida, id);
4393 }
4394
4395 static int its_vpe_init(struct its_vpe *vpe)
4396 {
4397         struct page *vpt_page;
4398         int vpe_id;
4399
4400         /* Allocate vpe_id */
4401         vpe_id = its_vpe_id_alloc();
4402         if (vpe_id < 0)
4403                 return vpe_id;
4404
4405         /* Allocate VPT */
4406         vpt_page = its_allocate_pending_table(GFP_KERNEL);
4407         if (!vpt_page) {
4408                 its_vpe_id_free(vpe_id);
4409                 return -ENOMEM;
4410         }
4411
4412         if (!its_alloc_vpe_table(vpe_id)) {
4413                 its_vpe_id_free(vpe_id);
4414                 its_free_pending_table(vpt_page);
4415                 return -ENOMEM;
4416         }
4417
4418         raw_spin_lock_init(&vpe->vpe_lock);
4419         vpe->vpe_id = vpe_id;
4420         vpe->vpt_page = vpt_page;
4421         if (gic_rdists->has_rvpeid)
4422                 atomic_set(&vpe->vmapp_count, 0);
4423         else
4424                 vpe->vpe_proxy_event = -1;
4425
4426         return 0;
4427 }
4428
4429 static void its_vpe_teardown(struct its_vpe *vpe)
4430 {
4431         its_vpe_db_proxy_unmap(vpe);
4432         its_vpe_id_free(vpe->vpe_id);
4433         its_free_pending_table(vpe->vpt_page);
4434 }
4435
4436 static void its_vpe_irq_domain_free(struct irq_domain *domain,
4437                                     unsigned int virq,
4438                                     unsigned int nr_irqs)
4439 {
4440         struct its_vm *vm = domain->host_data;
4441         int i;
4442
4443         irq_domain_free_irqs_parent(domain, virq, nr_irqs);
4444
4445         for (i = 0; i < nr_irqs; i++) {
4446                 struct irq_data *data = irq_domain_get_irq_data(domain,
4447                                                                 virq + i);
4448                 struct its_vpe *vpe = irq_data_get_irq_chip_data(data);
4449
4450                 BUG_ON(vm != vpe->its_vm);
4451
4452                 clear_bit(data->hwirq, vm->db_bitmap);
4453                 its_vpe_teardown(vpe);
4454                 irq_domain_reset_irq_data(data);
4455         }
4456
4457         if (bitmap_empty(vm->db_bitmap, vm->nr_db_lpis)) {
4458                 its_lpi_free(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis);
4459                 its_free_prop_table(vm->vprop_page);
4460         }
4461 }
4462
4463 static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
4464                                     unsigned int nr_irqs, void *args)
4465 {
4466         struct irq_chip *irqchip = &its_vpe_irq_chip;
4467         struct its_vm *vm = args;
4468         unsigned long *bitmap;
4469         struct page *vprop_page;
4470         int base, nr_ids, i, err = 0;
4471
4472         BUG_ON(!vm);
4473
4474         bitmap = its_lpi_alloc(roundup_pow_of_two(nr_irqs), &base, &nr_ids);
4475         if (!bitmap)
4476                 return -ENOMEM;
4477
4478         if (nr_ids < nr_irqs) {
4479                 its_lpi_free(bitmap, base, nr_ids);
4480                 return -ENOMEM;
4481         }
4482
4483         vprop_page = its_allocate_prop_table(GFP_KERNEL);
4484         if (!vprop_page) {
4485                 its_lpi_free(bitmap, base, nr_ids);
4486                 return -ENOMEM;
4487         }
4488
4489         vm->db_bitmap = bitmap;
4490         vm->db_lpi_base = base;
4491         vm->nr_db_lpis = nr_ids;
4492         vm->vprop_page = vprop_page;
4493
4494         if (gic_rdists->has_rvpeid)
4495                 irqchip = &its_vpe_4_1_irq_chip;
4496
4497         for (i = 0; i < nr_irqs; i++) {
4498                 vm->vpes[i]->vpe_db_lpi = base + i;
4499                 err = its_vpe_init(vm->vpes[i]);
4500                 if (err)
4501                         break;
4502                 err = its_irq_gic_domain_alloc(domain, virq + i,
4503                                                vm->vpes[i]->vpe_db_lpi);
4504                 if (err)
4505                         break;
4506                 irq_domain_set_hwirq_and_chip(domain, virq + i, i,
4507                                               irqchip, vm->vpes[i]);
4508                 set_bit(i, bitmap);
4509         }
4510
4511         if (err)
4512                 its_vpe_irq_domain_free(domain, virq, i);
4513
4514         return err;
4515 }
4516
4517 static int its_vpe_irq_domain_activate(struct irq_domain *domain,
4518                                        struct irq_data *d, bool reserve)
4519 {
4520         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4521         struct its_node *its;
4522
4523         /*
4524          * If we use the list map, we issue VMAPP on demand... Unless
4525          * we're on a GICv4.1 and we eagerly map the VPE on all ITSs
4526          * so that VSGIs can work.
4527          */
4528         if (!gic_requires_eager_mapping())
4529                 return 0;
4530
4531         /* Map the VPE to the first possible CPU */
4532         vpe->col_idx = cpumask_first(cpu_online_mask);
4533
4534         list_for_each_entry(its, &its_nodes, entry) {
4535                 if (!is_v4(its))
4536                         continue;
4537
4538                 its_send_vmapp(its, vpe, true);
4539                 its_send_vinvall(its, vpe);
4540         }
4541
4542         irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
4543
4544         return 0;
4545 }
4546
4547 static void its_vpe_irq_domain_deactivate(struct irq_domain *domain,
4548                                           struct irq_data *d)
4549 {
4550         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4551         struct its_node *its;
4552
4553         /*
4554          * If we use the list map on GICv4.0, we unmap the VPE once no
4555          * VLPIs are associated with the VM.
4556          */
4557         if (!gic_requires_eager_mapping())
4558                 return;
4559
4560         list_for_each_entry(its, &its_nodes, entry) {
4561                 if (!is_v4(its))
4562                         continue;
4563
4564                 its_send_vmapp(its, vpe, false);
4565         }
4566 }
4567
4568 static const struct irq_domain_ops its_vpe_domain_ops = {
4569         .alloc                  = its_vpe_irq_domain_alloc,
4570         .free                   = its_vpe_irq_domain_free,
4571         .activate               = its_vpe_irq_domain_activate,
4572         .deactivate             = its_vpe_irq_domain_deactivate,
4573 };
4574
4575 static int its_force_quiescent(void __iomem *base)
4576 {
4577         u32 count = 1000000;    /* 1s */
4578         u32 val;
4579
4580         val = readl_relaxed(base + GITS_CTLR);
4581         /*
4582          * GIC architecture specification requires the ITS to be both
4583          * disabled and quiescent for writes to GITS_BASER<n> or
4584          * GITS_CBASER to not have UNPREDICTABLE results.
4585          */
4586         if ((val & GITS_CTLR_QUIESCENT) && !(val & GITS_CTLR_ENABLE))
4587                 return 0;
4588
4589         /* Disable the generation of all interrupts to this ITS */
4590         val &= ~(GITS_CTLR_ENABLE | GITS_CTLR_ImDe);
4591         writel_relaxed(val, base + GITS_CTLR);
4592
4593         /* Poll GITS_CTLR and wait until ITS becomes quiescent */
4594         while (1) {
4595                 val = readl_relaxed(base + GITS_CTLR);
4596                 if (val & GITS_CTLR_QUIESCENT)
4597                         return 0;
4598
4599                 count--;
4600                 if (!count)
4601                         return -EBUSY;
4602
4603                 cpu_relax();
4604                 udelay(1);
4605         }
4606 }
4607
4608 static bool __maybe_unused its_enable_quirk_cavium_22375(void *data)
4609 {
4610         struct its_node *its = data;
4611
4612         /* erratum 22375: only alloc 8MB table size (20 bits) */
4613         its->typer &= ~GITS_TYPER_DEVBITS;
4614         its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1);
4615         its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375;
4616
4617         return true;
4618 }
4619
4620 static bool __maybe_unused its_enable_quirk_cavium_23144(void *data)
4621 {
4622         struct its_node *its = data;
4623
4624         its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144;
4625
4626         return true;
4627 }
4628
4629 static bool __maybe_unused its_enable_quirk_qdf2400_e0065(void *data)
4630 {
4631         struct its_node *its = data;
4632
4633         /* On QDF2400, the size of the ITE is 16Bytes */
4634         its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE;
4635         its->typer |= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1);
4636
4637         return true;
4638 }
4639
4640 static u64 its_irq_get_msi_base_pre_its(struct its_device *its_dev)
4641 {
4642         struct its_node *its = its_dev->its;
4643
4644         /*
4645          * The Socionext Synquacer SoC has a so-called 'pre-ITS',
4646          * which maps 32-bit writes targeted at a separate window of
4647          * size '4 << device_id_bits' onto writes to GITS_TRANSLATER
4648          * with device ID taken from bits [device_id_bits + 1:2] of
4649          * the window offset.
4650          */
4651         return its->pre_its_base + (its_dev->device_id << 2);
4652 }
4653
4654 static bool __maybe_unused its_enable_quirk_socionext_synquacer(void *data)
4655 {
4656         struct its_node *its = data;
4657         u32 pre_its_window[2];
4658         u32 ids;
4659
4660         if (!fwnode_property_read_u32_array(its->fwnode_handle,
4661                                            "socionext,synquacer-pre-its",
4662                                            pre_its_window,
4663                                            ARRAY_SIZE(pre_its_window))) {
4664
4665                 its->pre_its_base = pre_its_window[0];
4666                 its->get_msi_base = its_irq_get_msi_base_pre_its;
4667
4668                 ids = ilog2(pre_its_window[1]) - 2;
4669                 if (device_ids(its) > ids) {
4670                         its->typer &= ~GITS_TYPER_DEVBITS;
4671                         its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1);
4672                 }
4673
4674                 /* the pre-ITS breaks isolation, so disable MSI remapping */
4675                 its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_MSI_REMAP;
4676                 return true;
4677         }
4678         return false;
4679 }
4680
4681 static bool __maybe_unused its_enable_quirk_hip07_161600802(void *data)
4682 {
4683         struct its_node *its = data;
4684
4685         /*
4686          * Hip07 insists on using the wrong address for the VLPI
4687          * page. Trick it into doing the right thing...
4688          */
4689         its->vlpi_redist_offset = SZ_128K;
4690         return true;
4691 }
4692
4693 static const struct gic_quirk its_quirks[] = {
4694 #ifdef CONFIG_CAVIUM_ERRATUM_22375
4695         {
4696                 .desc   = "ITS: Cavium errata 22375, 24313",
4697                 .iidr   = 0xa100034c,   /* ThunderX pass 1.x */
4698                 .mask   = 0xffff0fff,
4699                 .init   = its_enable_quirk_cavium_22375,
4700         },
4701 #endif
4702 #ifdef CONFIG_CAVIUM_ERRATUM_23144
4703         {
4704                 .desc   = "ITS: Cavium erratum 23144",
4705                 .iidr   = 0xa100034c,   /* ThunderX pass 1.x */
4706                 .mask   = 0xffff0fff,
4707                 .init   = its_enable_quirk_cavium_23144,
4708         },
4709 #endif
4710 #ifdef CONFIG_QCOM_QDF2400_ERRATUM_0065
4711         {
4712                 .desc   = "ITS: QDF2400 erratum 0065",
4713                 .iidr   = 0x00001070, /* QDF2400 ITS rev 1.x */
4714                 .mask   = 0xffffffff,
4715                 .init   = its_enable_quirk_qdf2400_e0065,
4716         },
4717 #endif
4718 #ifdef CONFIG_SOCIONEXT_SYNQUACER_PREITS
4719         {
4720                 /*
4721                  * The Socionext Synquacer SoC incorporates ARM's own GIC-500
4722                  * implementation, but with a 'pre-ITS' added that requires
4723                  * special handling in software.
4724                  */
4725                 .desc   = "ITS: Socionext Synquacer pre-ITS",
4726                 .iidr   = 0x0001143b,
4727                 .mask   = 0xffffffff,
4728                 .init   = its_enable_quirk_socionext_synquacer,
4729         },
4730 #endif
4731 #ifdef CONFIG_HISILICON_ERRATUM_161600802
4732         {
4733                 .desc   = "ITS: Hip07 erratum 161600802",
4734                 .iidr   = 0x00000004,
4735                 .mask   = 0xffffffff,
4736                 .init   = its_enable_quirk_hip07_161600802,
4737         },
4738 #endif
4739         {
4740         }
4741 };
4742
4743 static void its_enable_quirks(struct its_node *its)
4744 {
4745         u32 iidr = readl_relaxed(its->base + GITS_IIDR);
4746
4747         gic_enable_quirks(iidr, its_quirks, its);
4748 }
4749
4750 static int its_save_disable(void)
4751 {
4752         struct its_node *its;
4753         int err = 0;
4754
4755         raw_spin_lock(&its_lock);
4756         list_for_each_entry(its, &its_nodes, entry) {
4757                 void __iomem *base;
4758
4759                 base = its->base;
4760                 its->ctlr_save = readl_relaxed(base + GITS_CTLR);
4761                 err = its_force_quiescent(base);
4762                 if (err) {
4763                         pr_err("ITS@%pa: failed to quiesce: %d\n",
4764                                &its->phys_base, err);
4765                         writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4766                         goto err;
4767                 }
4768
4769                 its->cbaser_save = gits_read_cbaser(base + GITS_CBASER);
4770         }
4771
4772 err:
4773         if (err) {
4774                 list_for_each_entry_continue_reverse(its, &its_nodes, entry) {
4775                         void __iomem *base;
4776
4777                         base = its->base;
4778                         writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4779                 }
4780         }
4781         raw_spin_unlock(&its_lock);
4782
4783         return err;
4784 }
4785
4786 static void its_restore_enable(void)
4787 {
4788         struct its_node *its;
4789         int ret;
4790
4791         raw_spin_lock(&its_lock);
4792         list_for_each_entry(its, &its_nodes, entry) {
4793                 void __iomem *base;
4794                 int i;
4795
4796                 base = its->base;
4797
4798                 /*
4799                  * Make sure that the ITS is disabled. If it fails to quiesce,
4800                  * don't restore it since writing to CBASER or BASER<n>
4801                  * registers is undefined according to the GIC v3 ITS
4802                  * Specification.
4803                  *
4804                  * Firmware resuming with the ITS enabled is terminally broken.
4805                  */
4806                 WARN_ON(readl_relaxed(base + GITS_CTLR) & GITS_CTLR_ENABLE);
4807                 ret = its_force_quiescent(base);
4808                 if (ret) {
4809                         pr_err("ITS@%pa: failed to quiesce on resume: %d\n",
4810                                &its->phys_base, ret);
4811                         continue;
4812                 }
4813
4814                 gits_write_cbaser(its->cbaser_save, base + GITS_CBASER);
4815
4816                 /*
4817                  * Writing CBASER resets CREADR to 0, so make CWRITER and
4818                  * cmd_write line up with it.
4819                  */
4820                 its->cmd_write = its->cmd_base;
4821                 gits_write_cwriter(0, base + GITS_CWRITER);
4822
4823                 /* Restore GITS_BASER from the value cache. */
4824                 for (i = 0; i < GITS_BASER_NR_REGS; i++) {
4825                         struct its_baser *baser = &its->tables[i];
4826
4827                         if (!(baser->val & GITS_BASER_VALID))
4828                                 continue;
4829
4830                         its_write_baser(its, baser, baser->val);
4831                 }
4832                 writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4833
4834                 /*
4835                  * Reinit the collection if it's stored in the ITS. This is
4836                  * indicated by the col_id being less than the HCC field.
4837                  * CID < HCC as specified in the GIC v3 Documentation.
4838                  */
4839                 if (its->collections[smp_processor_id()].col_id <
4840                     GITS_TYPER_HCC(gic_read_typer(base + GITS_TYPER)))
4841                         its_cpu_init_collection(its);
4842         }
4843         raw_spin_unlock(&its_lock);
4844 }
4845
4846 static struct syscore_ops its_syscore_ops = {
4847         .suspend = its_save_disable,
4848         .resume = its_restore_enable,
4849 };
4850
4851 static int its_init_domain(struct fwnode_handle *handle, struct its_node *its)
4852 {
4853         struct irq_domain *inner_domain;
4854         struct msi_domain_info *info;
4855
4856         info = kzalloc(sizeof(*info), GFP_KERNEL);
4857         if (!info)
4858                 return -ENOMEM;
4859
4860         inner_domain = irq_domain_create_tree(handle, &its_domain_ops, its);
4861         if (!inner_domain) {
4862                 kfree(info);
4863                 return -ENOMEM;
4864         }
4865
4866         inner_domain->parent = its_parent;
4867         irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS);
4868         inner_domain->flags |= its->msi_domain_flags;
4869         info->ops = &its_msi_domain_ops;
4870         info->data = its;
4871         inner_domain->host_data = info;
4872
4873         return 0;
4874 }
4875
4876 static int its_init_vpe_domain(void)
4877 {
4878         struct its_node *its;
4879         u32 devid;
4880         int entries;
4881
4882         if (gic_rdists->has_direct_lpi) {
4883                 pr_info("ITS: Using DirectLPI for VPE invalidation\n");
4884                 return 0;
4885         }
4886
4887         /* Any ITS will do, even if not v4 */
4888         its = list_first_entry(&its_nodes, struct its_node, entry);
4889
4890         entries = roundup_pow_of_two(nr_cpu_ids);
4891         vpe_proxy.vpes = kcalloc(entries, sizeof(*vpe_proxy.vpes),
4892                                  GFP_KERNEL);
4893         if (!vpe_proxy.vpes) {
4894                 pr_err("ITS: Can't allocate GICv4 proxy device array\n");
4895                 return -ENOMEM;
4896         }
4897
4898         /* Use the last possible DevID */
4899         devid = GENMASK(device_ids(its) - 1, 0);
4900         vpe_proxy.dev = its_create_device(its, devid, entries, false);
4901         if (!vpe_proxy.dev) {
4902                 kfree(vpe_proxy.vpes);
4903                 pr_err("ITS: Can't allocate GICv4 proxy device\n");
4904                 return -ENOMEM;
4905         }
4906
4907         BUG_ON(entries > vpe_proxy.dev->nr_ites);
4908
4909         raw_spin_lock_init(&vpe_proxy.lock);
4910         vpe_proxy.next_victim = 0;
4911         pr_info("ITS: Allocated DevID %x as GICv4 proxy device (%d slots)\n",
4912                 devid, vpe_proxy.dev->nr_ites);
4913
4914         return 0;
4915 }
4916
4917 static int __init its_compute_its_list_map(struct resource *res,
4918                                            void __iomem *its_base)
4919 {
4920         int its_number;
4921         u32 ctlr;
4922
4923         /*
4924          * This is assumed to be done early enough that we're
4925          * guaranteed to be single-threaded, hence no
4926          * locking. Should this change, we should address
4927          * this.
4928          */
4929         its_number = find_first_zero_bit(&its_list_map, GICv4_ITS_LIST_MAX);
4930         if (its_number >= GICv4_ITS_LIST_MAX) {
4931                 pr_err("ITS@%pa: No ITSList entry available!\n",
4932                        &res->start);
4933                 return -EINVAL;
4934         }
4935
4936         ctlr = readl_relaxed(its_base + GITS_CTLR);
4937         ctlr &= ~GITS_CTLR_ITS_NUMBER;
4938         ctlr |= its_number << GITS_CTLR_ITS_NUMBER_SHIFT;
4939         writel_relaxed(ctlr, its_base + GITS_CTLR);
4940         ctlr = readl_relaxed(its_base + GITS_CTLR);
4941         if ((ctlr & GITS_CTLR_ITS_NUMBER) != (its_number << GITS_CTLR_ITS_NUMBER_SHIFT)) {
4942                 its_number = ctlr & GITS_CTLR_ITS_NUMBER;
4943                 its_number >>= GITS_CTLR_ITS_NUMBER_SHIFT;
4944         }
4945
4946         if (test_and_set_bit(its_number, &its_list_map)) {
4947                 pr_err("ITS@%pa: Duplicate ITSList entry %d\n",
4948                        &res->start, its_number);
4949                 return -EINVAL;
4950         }
4951
4952         return its_number;
4953 }
4954
4955 static int __init its_probe_one(struct resource *res,
4956                                 struct fwnode_handle *handle, int numa_node)
4957 {
4958         struct its_node *its;
4959         void __iomem *its_base;
4960         u32 val, ctlr;
4961         u64 baser, tmp, typer;
4962         struct page *page;
4963         int err;
4964
4965         its_base = ioremap(res->start, SZ_64K);
4966         if (!its_base) {
4967                 pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start);
4968                 return -ENOMEM;
4969         }
4970
4971         val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK;
4972         if (val != 0x30 && val != 0x40) {
4973                 pr_warn("ITS@%pa: No ITS detected, giving up\n", &res->start);
4974                 err = -ENODEV;
4975                 goto out_unmap;
4976         }
4977
4978         err = its_force_quiescent(its_base);
4979         if (err) {
4980                 pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res->start);
4981                 goto out_unmap;
4982         }
4983
4984         pr_info("ITS %pR\n", res);
4985
4986         its = kzalloc(sizeof(*its), GFP_KERNEL);
4987         if (!its) {
4988                 err = -ENOMEM;
4989                 goto out_unmap;
4990         }
4991
4992         raw_spin_lock_init(&its->lock);
4993         mutex_init(&its->dev_alloc_lock);
4994         INIT_LIST_HEAD(&its->entry);
4995         INIT_LIST_HEAD(&its->its_device_list);
4996         typer = gic_read_typer(its_base + GITS_TYPER);
4997         its->typer = typer;
4998         its->base = its_base;
4999         its->phys_base = res->start;
5000         if (is_v4(its)) {
5001                 if (!(typer & GITS_TYPER_VMOVP)) {
5002                         err = its_compute_its_list_map(res, its_base);
5003                         if (err < 0)
5004                                 goto out_free_its;
5005
5006                         its->list_nr = err;
5007
5008                         pr_info("ITS@%pa: Using ITS number %d\n",
5009                                 &res->start, err);
5010                 } else {
5011                         pr_info("ITS@%pa: Single VMOVP capable\n", &res->start);
5012                 }
5013
5014                 if (is_v4_1(its)) {
5015                         u32 svpet = FIELD_GET(GITS_TYPER_SVPET, typer);
5016
5017                         its->sgir_base = ioremap(res->start + SZ_128K, SZ_64K);
5018                         if (!its->sgir_base) {
5019                                 err = -ENOMEM;
5020                                 goto out_free_its;
5021                         }
5022
5023                         its->mpidr = readl_relaxed(its_base + GITS_MPIDR);
5024
5025                         pr_info("ITS@%pa: Using GICv4.1 mode %08x %08x\n",
5026                                 &res->start, its->mpidr, svpet);
5027                 }
5028         }
5029
5030         its->numa_node = numa_node;
5031
5032         page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
5033                                 get_order(ITS_CMD_QUEUE_SZ));
5034         if (!page) {
5035                 err = -ENOMEM;
5036                 goto out_unmap_sgir;
5037         }
5038         its->cmd_base = (void *)page_address(page);
5039         its->cmd_write = its->cmd_base;
5040         its->fwnode_handle = handle;
5041         its->get_msi_base = its_irq_get_msi_base;
5042         its->msi_domain_flags = IRQ_DOMAIN_FLAG_MSI_REMAP;
5043
5044         its_enable_quirks(its);
5045
5046         err = its_alloc_tables(its);
5047         if (err)
5048                 goto out_free_cmd;
5049
5050         err = its_alloc_collections(its);
5051         if (err)
5052                 goto out_free_tables;
5053
5054         baser = (virt_to_phys(its->cmd_base)    |
5055                  GITS_CBASER_RaWaWb             |
5056                  GITS_CBASER_InnerShareable     |
5057                  (ITS_CMD_QUEUE_SZ / SZ_4K - 1) |
5058                  GITS_CBASER_VALID);
5059
5060         gits_write_cbaser(baser, its->base + GITS_CBASER);
5061         tmp = gits_read_cbaser(its->base + GITS_CBASER);
5062
5063         if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) {
5064                 if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) {
5065                         /*
5066                          * The HW reports non-shareable, we must
5067                          * remove the cacheability attributes as
5068                          * well.
5069                          */
5070                         baser &= ~(GITS_CBASER_SHAREABILITY_MASK |
5071                                    GITS_CBASER_CACHEABILITY_MASK);
5072                         baser |= GITS_CBASER_nC;
5073                         gits_write_cbaser(baser, its->base + GITS_CBASER);
5074                 }
5075                 pr_info("ITS: using cache flushing for cmd queue\n");
5076                 its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING;
5077         }
5078
5079         gits_write_cwriter(0, its->base + GITS_CWRITER);
5080         ctlr = readl_relaxed(its->base + GITS_CTLR);
5081         ctlr |= GITS_CTLR_ENABLE;
5082         if (is_v4(its))
5083                 ctlr |= GITS_CTLR_ImDe;
5084         writel_relaxed(ctlr, its->base + GITS_CTLR);
5085
5086         err = its_init_domain(handle, its);
5087         if (err)
5088                 goto out_free_tables;
5089
5090         raw_spin_lock(&its_lock);
5091         list_add(&its->entry, &its_nodes);
5092         raw_spin_unlock(&its_lock);
5093
5094         return 0;
5095
5096 out_free_tables:
5097         its_free_tables(its);
5098 out_free_cmd:
5099         free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ));
5100 out_unmap_sgir:
5101         if (its->sgir_base)
5102                 iounmap(its->sgir_base);
5103 out_free_its:
5104         kfree(its);
5105 out_unmap:
5106         iounmap(its_base);
5107         pr_err("ITS@%pa: failed probing (%d)\n", &res->start, err);
5108         return err;
5109 }
5110
5111 static bool gic_rdists_supports_plpis(void)
5112 {
5113         return !!(gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS);
5114 }
5115
5116 static int redist_disable_lpis(void)
5117 {
5118         void __iomem *rbase = gic_data_rdist_rd_base();
5119         u64 timeout = USEC_PER_SEC;
5120         u64 val;
5121
5122         if (!gic_rdists_supports_plpis()) {
5123                 pr_info("CPU%d: LPIs not supported\n", smp_processor_id());
5124                 return -ENXIO;
5125         }
5126
5127         val = readl_relaxed(rbase + GICR_CTLR);
5128         if (!(val & GICR_CTLR_ENABLE_LPIS))
5129                 return 0;
5130
5131         /*
5132          * If coming via a CPU hotplug event, we don't need to disable
5133          * LPIs before trying to re-enable them. They are already
5134          * configured and all is well in the world.
5135          *
5136          * If running with preallocated tables, there is nothing to do.
5137          */
5138         if (gic_data_rdist()->lpi_enabled ||
5139             (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED))
5140                 return 0;
5141
5142         /*
5143          * From that point on, we only try to do some damage control.
5144          */
5145         pr_warn("GICv3: CPU%d: Booted with LPIs enabled, memory probably corrupted\n",
5146                 smp_processor_id());
5147         add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
5148
5149         /* Disable LPIs */
5150         val &= ~GICR_CTLR_ENABLE_LPIS;
5151         writel_relaxed(val, rbase + GICR_CTLR);
5152
5153         /* Make sure any change to GICR_CTLR is observable by the GIC */
5154         dsb(sy);
5155
5156         /*
5157          * Software must observe RWP==0 after clearing GICR_CTLR.EnableLPIs
5158          * from 1 to 0 before programming GICR_PEND{PROP}BASER registers.
5159          * Error out if we time out waiting for RWP to clear.
5160          */
5161         while (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_RWP) {
5162                 if (!timeout) {
5163                         pr_err("CPU%d: Timeout while disabling LPIs\n",
5164                                smp_processor_id());
5165                         return -ETIMEDOUT;
5166                 }
5167                 udelay(1);
5168                 timeout--;
5169         }
5170
5171         /*
5172          * After it has been written to 1, it is IMPLEMENTATION
5173          * DEFINED whether GICR_CTLR.EnableLPI becomes RES1 or can be
5174          * cleared to 0. Error out if clearing the bit failed.
5175          */
5176         if (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_ENABLE_LPIS) {
5177                 pr_err("CPU%d: Failed to disable LPIs\n", smp_processor_id());
5178                 return -EBUSY;
5179         }
5180
5181         return 0;
5182 }
5183
5184 int its_cpu_init(void)
5185 {
5186         if (!list_empty(&its_nodes)) {
5187                 int ret;
5188
5189                 ret = redist_disable_lpis();
5190                 if (ret)
5191                         return ret;
5192
5193                 its_cpu_init_lpis();
5194                 its_cpu_init_collections();
5195         }
5196
5197         return 0;
5198 }
5199
5200 static const struct of_device_id its_device_id[] = {
5201         {       .compatible     = "arm,gic-v3-its",     },
5202         {},
5203 };
5204
5205 static int __init its_of_probe(struct device_node *node)
5206 {
5207         struct device_node *np;
5208         struct resource res;
5209
5210         for (np = of_find_matching_node(node, its_device_id); np;
5211              np = of_find_matching_node(np, its_device_id)) {
5212                 if (!of_device_is_available(np))
5213                         continue;
5214                 if (!of_property_read_bool(np, "msi-controller")) {
5215                         pr_warn("%pOF: no msi-controller property, ITS ignored\n",
5216                                 np);
5217                         continue;
5218                 }
5219
5220                 if (of_address_to_resource(np, 0, &res)) {
5221                         pr_warn("%pOF: no regs?\n", np);
5222                         continue;
5223                 }
5224
5225                 its_probe_one(&res, &np->fwnode, of_node_to_nid(np));
5226         }
5227         return 0;
5228 }
5229
5230 #ifdef CONFIG_ACPI
5231
5232 #define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K)
5233
5234 #ifdef CONFIG_ACPI_NUMA
5235 struct its_srat_map {
5236         /* numa node id */
5237         u32     numa_node;
5238         /* GIC ITS ID */
5239         u32     its_id;
5240 };
5241
5242 static struct its_srat_map *its_srat_maps __initdata;
5243 static int its_in_srat __initdata;
5244
5245 static int __init acpi_get_its_numa_node(u32 its_id)
5246 {
5247         int i;
5248
5249         for (i = 0; i < its_in_srat; i++) {
5250                 if (its_id == its_srat_maps[i].its_id)
5251                         return its_srat_maps[i].numa_node;
5252         }
5253         return NUMA_NO_NODE;
5254 }
5255
5256 static int __init gic_acpi_match_srat_its(union acpi_subtable_headers *header,
5257                                           const unsigned long end)
5258 {
5259         return 0;
5260 }
5261
5262 static int __init gic_acpi_parse_srat_its(union acpi_subtable_headers *header,
5263                          const unsigned long end)
5264 {
5265         int node;
5266         struct acpi_srat_gic_its_affinity *its_affinity;
5267
5268         its_affinity = (struct acpi_srat_gic_its_affinity *)header;
5269         if (!its_affinity)
5270                 return -EINVAL;
5271
5272         if (its_affinity->header.length < sizeof(*its_affinity)) {
5273                 pr_err("SRAT: Invalid header length %d in ITS affinity\n",
5274                         its_affinity->header.length);
5275                 return -EINVAL;
5276         }
5277
5278         /*
5279          * Note that in theory a new proximity node could be created by this
5280          * entry as it is an SRAT resource allocation structure.
5281          * We do not currently support doing so.
5282          */
5283         node = pxm_to_node(its_affinity->proximity_domain);
5284
5285         if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) {
5286                 pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node);
5287                 return 0;
5288         }
5289
5290         its_srat_maps[its_in_srat].numa_node = node;
5291         its_srat_maps[its_in_srat].its_id = its_affinity->its_id;
5292         its_in_srat++;
5293         pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n",
5294                 its_affinity->proximity_domain, its_affinity->its_id, node);
5295
5296         return 0;
5297 }
5298
5299 static void __init acpi_table_parse_srat_its(void)
5300 {
5301         int count;
5302
5303         count = acpi_table_parse_entries(ACPI_SIG_SRAT,
5304                         sizeof(struct acpi_table_srat),
5305                         ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
5306                         gic_acpi_match_srat_its, 0);
5307         if (count <= 0)
5308                 return;
5309
5310         its_srat_maps = kmalloc_array(count, sizeof(struct its_srat_map),
5311                                       GFP_KERNEL);
5312         if (!its_srat_maps) {
5313                 pr_warn("SRAT: Failed to allocate memory for its_srat_maps!\n");
5314                 return;
5315         }
5316
5317         acpi_table_parse_entries(ACPI_SIG_SRAT,
5318                         sizeof(struct acpi_table_srat),
5319                         ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
5320                         gic_acpi_parse_srat_its, 0);
5321 }
5322
5323 /* free the its_srat_maps after ITS probing */
5324 static void __init acpi_its_srat_maps_free(void)
5325 {
5326         kfree(its_srat_maps);
5327 }
5328 #else
5329 static void __init acpi_table_parse_srat_its(void)      { }
5330 static int __init acpi_get_its_numa_node(u32 its_id) { return NUMA_NO_NODE; }
5331 static void __init acpi_its_srat_maps_free(void) { }
5332 #endif
5333
5334 static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header,
5335                                           const unsigned long end)
5336 {
5337         struct acpi_madt_generic_translator *its_entry;
5338         struct fwnode_handle *dom_handle;
5339         struct resource res;
5340         int err;
5341
5342         its_entry = (struct acpi_madt_generic_translator *)header;
5343         memset(&res, 0, sizeof(res));
5344         res.start = its_entry->base_address;
5345         res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1;
5346         res.flags = IORESOURCE_MEM;
5347
5348         dom_handle = irq_domain_alloc_fwnode(&res.start);
5349         if (!dom_handle) {
5350                 pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n",
5351                        &res.start);
5352                 return -ENOMEM;
5353         }
5354
5355         err = iort_register_domain_token(its_entry->translation_id, res.start,
5356                                          dom_handle);
5357         if (err) {
5358                 pr_err("ITS@%pa: Unable to register GICv3 ITS domain token (ITS ID %d) to IORT\n",
5359                        &res.start, its_entry->translation_id);
5360                 goto dom_err;
5361         }
5362
5363         err = its_probe_one(&res, dom_handle,
5364                         acpi_get_its_numa_node(its_entry->translation_id));
5365         if (!err)
5366                 return 0;
5367
5368         iort_deregister_domain_token(its_entry->translation_id);
5369 dom_err:
5370         irq_domain_free_fwnode(dom_handle);
5371         return err;
5372 }
5373
5374 static void __init its_acpi_probe(void)
5375 {
5376         acpi_table_parse_srat_its();
5377         acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
5378                               gic_acpi_parse_madt_its, 0);
5379         acpi_its_srat_maps_free();
5380 }
5381 #else
5382 static void __init its_acpi_probe(void) { }
5383 #endif
5384
5385 int __init its_init(struct fwnode_handle *handle, struct rdists *rdists,
5386                     struct irq_domain *parent_domain)
5387 {
5388         struct device_node *of_node;
5389         struct its_node *its;
5390         bool has_v4 = false;
5391         bool has_v4_1 = false;
5392         int err;
5393
5394         gic_rdists = rdists;
5395
5396         its_parent = parent_domain;
5397         of_node = to_of_node(handle);
5398         if (of_node)
5399                 its_of_probe(of_node);
5400         else
5401                 its_acpi_probe();
5402
5403         if (list_empty(&its_nodes)) {
5404                 pr_warn("ITS: No ITS available, not enabling LPIs\n");
5405                 return -ENXIO;
5406         }
5407
5408         err = allocate_lpi_tables();
5409         if (err)
5410                 return err;
5411
5412         list_for_each_entry(its, &its_nodes, entry) {
5413                 has_v4 |= is_v4(its);
5414                 has_v4_1 |= is_v4_1(its);
5415         }
5416
5417         /* Don't bother with inconsistent systems */
5418         if (WARN_ON(!has_v4_1 && rdists->has_rvpeid))
5419                 rdists->has_rvpeid = false;
5420
5421         if (has_v4 & rdists->has_vlpis) {
5422                 const struct irq_domain_ops *sgi_ops;
5423
5424                 if (has_v4_1)
5425                         sgi_ops = &its_sgi_domain_ops;
5426                 else
5427                         sgi_ops = NULL;
5428
5429                 if (its_init_vpe_domain() ||
5430                     its_init_v4(parent_domain, &its_vpe_domain_ops, sgi_ops)) {
5431                         rdists->has_vlpis = false;
5432                         pr_err("ITS: Disabling GICv4 support\n");
5433                 }
5434         }
5435
5436         register_syscore_ops(&its_syscore_ops);
5437
5438         return 0;
5439 }