drivers/iommu/arm/arm-smmu/arm-smmu.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * IOMMU API for ARM architected SMMU implementations.
   4  *
   5  * Copyright (C) 2013 ARM Limited
   6  *
   7  * Author: Will Deacon <will.deacon@arm.com>
   8  *
   9  * This driver currently supports:
  10  *      - SMMUv1 and v2 implementations
  11  *      - Stream-matching and stream-indexing
  12  *      - v7/v8 long-descriptor format
  13  *      - Non-secure access to the SMMU
  14  *      - Context fault reporting
  15  *      - Extended Stream ID (16 bit)
  16  */
  17
  18 #define pr_fmt(fmt) "arm-smmu: " fmt
  19
  20 #include <linux/acpi.h>
  21 #include <linux/acpi_iort.h>
  22 #include <linux/bitfield.h>
  23 #include <linux/delay.h>
  24 #include <linux/dma-iommu.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/err.h>
  27 #include <linux/interrupt.h>
  28 #include <linux/io.h>
  29 #include <linux/iopoll.h>
  30 #include <linux/module.h>
  31 #include <linux/of.h>
  32 #include <linux/of_address.h>
  33 #include <linux/of_device.h>
  34 #include <linux/of_iommu.h>
  35 #include <linux/pci.h>
  36 #include <linux/platform_device.h>
  37 #include <linux/pm_runtime.h>
  38 #include <linux/ratelimit.h>
  39 #include <linux/slab.h>
  40
  41 #include <linux/amba/bus.h>
  42 #include <linux/fsl/mc.h>
  43
  44 #include "arm-smmu.h"
  45
  46 /*
  47  * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
  48  * global register space are still, in fact, using a hypervisor to mediate it
  49  * by trapping and emulating register accesses. Sadly, some deployed versions
  50  * of said trapping code have bugs wherein they go horribly wrong for stores
  51  * using r31 (i.e. XZR/WZR) as the source register.
  52  */
  53 #define QCOM_DUMMY_VAL -1
  54
  55 #define MSI_IOVA_BASE                   0x8000000
  56 #define MSI_IOVA_LENGTH                 0x100000
  57
  58 static int force_stage;
  59 module_param(force_stage, int, S_IRUGO);
  60 MODULE_PARM_DESC(force_stage,
  61         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
  62 static bool disable_bypass =
  63         IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
  64 module_param(disable_bypass, bool, S_IRUGO);
  65 MODULE_PARM_DESC(disable_bypass,
  66         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
  67
  68 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
  69         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
  70 }
  71
  72 static bool using_legacy_binding, using_generic_binding;
  73
  74 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
  75 {
  76         if (pm_runtime_enabled(smmu->dev))
  77                 return pm_runtime_resume_and_get(smmu->dev);
  78
  79         return 0;
  80 }
  81
  82 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
  83 {
  84         if (pm_runtime_enabled(smmu->dev))
  85                 pm_runtime_put_autosuspend(smmu->dev);
  86 }
  87
  88 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
  89 {
  90         return container_of(dom, struct arm_smmu_domain, domain);
  91 }
  92
  93 static struct platform_driver arm_smmu_driver;
  94 static struct iommu_ops arm_smmu_ops;
  95
  96 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
  97 static int arm_smmu_bus_init(struct iommu_ops *ops);
  98
  99 static struct device_node *dev_get_dev_node(struct device *dev)
 100 {
 101         if (dev_is_pci(dev)) {
 102                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 103
 104                 while (!pci_is_root_bus(bus))
 105                         bus = bus->parent;
 106                 return of_node_get(bus->bridge->parent->of_node);
 107         }
 108
 109         return of_node_get(dev->of_node);
 110 }
 111
 112 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 113 {
 114         *((__be32 *)data) = cpu_to_be32(alias);
 115         return 0; /* Continue walking */
 116 }
 117
 118 static int __find_legacy_master_phandle(struct device *dev, void *data)
 119 {
 120         struct of_phandle_iterator *it = *(void **)data;
 121         struct device_node *np = it->node;
 122         int err;
 123
 124         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
 125                             "#stream-id-cells", -1)
 126                 if (it->node == np) {
 127                         *(void **)data = dev;
 128                         return 1;
 129                 }
 130         it->node = np;
 131         return err == -ENOENT ? 0 : err;
 132 }
 133
 134 static int arm_smmu_register_legacy_master(struct device *dev,
 135                                            struct arm_smmu_device **smmu)
 136 {
 137         struct device *smmu_dev;
 138         struct device_node *np;
 139         struct of_phandle_iterator it;
 140         void *data = &it;
 141         u32 *sids;
 142         __be32 pci_sid;
 143         int err;
 144
 145         np = dev_get_dev_node(dev);
 146         if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
 147                 of_node_put(np);
 148                 return -ENODEV;
 149         }
 150
 151         it.node = np;
 152         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
 153                                      __find_legacy_master_phandle);
 154         smmu_dev = data;
 155         of_node_put(np);
 156         if (err == 0)
 157                 return -ENODEV;
 158         if (err < 0)
 159                 return err;
 160
 161         if (dev_is_pci(dev)) {
 162                 /* "mmu-masters" assumes Stream ID == Requester ID */
 163                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
 164                                        &pci_sid);
 165                 it.cur = &pci_sid;
 166                 it.cur_count = 1;
 167         }
 168
 169         err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
 170                                 &arm_smmu_ops);
 171         if (err)
 172                 return err;
 173
 174         sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
 175         if (!sids)
 176                 return -ENOMEM;
 177
 178         *smmu = dev_get_drvdata(smmu_dev);
 179         of_phandle_iterator_args(&it, sids, it.cur_count);
 180         err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
 181         kfree(sids);
 182         return err;
 183 }
 184
 185 /*
 186  * With the legacy DT binding in play, we have no guarantees about
 187  * probe order, but then we're also not doing default domains, so we can
 188  * delay setting bus ops until we're sure every possible SMMU is ready,
 189  * and that way ensure that no probe_device() calls get missed.
 190  */
 191 static int arm_smmu_legacy_bus_init(void)
 192 {
 193         if (using_legacy_binding)
 194                 return arm_smmu_bus_init(&arm_smmu_ops);
 195         return 0;
 196 }
 197 device_initcall_sync(arm_smmu_legacy_bus_init);
 198 #else
 199 static int arm_smmu_register_legacy_master(struct device *dev,
 200                                            struct arm_smmu_device **smmu)
 201 {
 202         return -ENODEV;
 203 }
 204 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
 205
 206 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 207 {
 208         clear_bit(idx, map);
 209 }
 210
 211 /* Wait for any pending TLB invalidations to complete */
 212 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
 213                                 int sync, int status)
 214 {
 215         unsigned int spin_cnt, delay;
 216         u32 reg;
 217
 218         if (smmu->impl && unlikely(smmu->impl->tlb_sync))
 219                 return smmu->impl->tlb_sync(smmu, page, sync, status);
 220
 221         arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
 222         for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
 223                 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
 224                         reg = arm_smmu_readl(smmu, page, status);
 225                         if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
 226                                 return;
 227                         cpu_relax();
 228                 }
 229                 udelay(delay);
 230         }
 231         dev_err_ratelimited(smmu->dev,
 232                             "TLB sync timed out -- SMMU may be deadlocked\n");
 233 }
 234
 235 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
 236 {
 237         unsigned long flags;
 238
 239         spin_lock_irqsave(&smmu->global_sync_lock, flags);
 240         __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
 241                             ARM_SMMU_GR0_sTLBGSTATUS);
 242         spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
 243 }
 244
 245 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
 246 {
 247         struct arm_smmu_device *smmu = smmu_domain->smmu;
 248         unsigned long flags;
 249
 250         spin_lock_irqsave(&smmu_domain->cb_lock, flags);
 251         __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
 252                             ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
 253         spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
 254 }
 255
 256 static void arm_smmu_tlb_inv_context_s1(void *cookie)
 257 {
 258         struct arm_smmu_domain *smmu_domain = cookie;
 259         /*
 260          * The TLBI write may be relaxed, so ensure that PTEs cleared by the
 261          * current CPU are visible beforehand.
 262          */
 263         wmb();
 264         arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
 265                           ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
 266         arm_smmu_tlb_sync_context(smmu_domain);
 267 }
 268
 269 static void arm_smmu_tlb_inv_context_s2(void *cookie)
 270 {
 271         struct arm_smmu_domain *smmu_domain = cookie;
 272         struct arm_smmu_device *smmu = smmu_domain->smmu;
 273
 274         /* See above */
 275         wmb();
 276         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
 277         arm_smmu_tlb_sync_global(smmu);
 278 }
 279
 280 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
 281                                       size_t granule, void *cookie, int reg)
 282 {
 283         struct arm_smmu_domain *smmu_domain = cookie;
 284         struct arm_smmu_device *smmu = smmu_domain->smmu;
 285         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 286         int idx = cfg->cbndx;
 287
 288         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 289                 wmb();
 290
 291         if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 292                 iova = (iova >> 12) << 12;
 293                 iova |= cfg->asid;
 294                 do {
 295                         arm_smmu_cb_write(smmu, idx, reg, iova);
 296                         iova += granule;
 297                 } while (size -= granule);
 298         } else {
 299                 iova >>= 12;
 300                 iova |= (u64)cfg->asid << 48;
 301                 do {
 302                         arm_smmu_cb_writeq(smmu, idx, reg, iova);
 303                         iova += granule >> 12;
 304                 } while (size -= granule);
 305         }
 306 }
 307
 308 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
 309                                       size_t granule, void *cookie, int reg)
 310 {
 311         struct arm_smmu_domain *smmu_domain = cookie;
 312         struct arm_smmu_device *smmu = smmu_domain->smmu;
 313         int idx = smmu_domain->cfg.cbndx;
 314
 315         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 316                 wmb();
 317
 318         iova >>= 12;
 319         do {
 320                 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
 321                         arm_smmu_cb_writeq(smmu, idx, reg, iova);
 322                 else
 323                         arm_smmu_cb_write(smmu, idx, reg, iova);
 324                 iova += granule >> 12;
 325         } while (size -= granule);
 326 }
 327
 328 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
 329                                      size_t granule, void *cookie)
 330 {
 331         arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
 332                                   ARM_SMMU_CB_S1_TLBIVA);
 333         arm_smmu_tlb_sync_context(cookie);
 334 }
 335
 336 static void arm_smmu_tlb_inv_leaf_s1(unsigned long iova, size_t size,
 337                                      size_t granule, void *cookie)
 338 {
 339         arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
 340                                   ARM_SMMU_CB_S1_TLBIVAL);
 341         arm_smmu_tlb_sync_context(cookie);
 342 }
 343
 344 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
 345                                      unsigned long iova, size_t granule,
 346                                      void *cookie)
 347 {
 348         arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
 349                                   ARM_SMMU_CB_S1_TLBIVAL);
 350 }
 351
 352 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
 353                                      size_t granule, void *cookie)
 354 {
 355         arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
 356                                   ARM_SMMU_CB_S2_TLBIIPAS2);
 357         arm_smmu_tlb_sync_context(cookie);
 358 }
 359
 360 static void arm_smmu_tlb_inv_leaf_s2(unsigned long iova, size_t size,
 361                                      size_t granule, void *cookie)
 362 {
 363         arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
 364                                   ARM_SMMU_CB_S2_TLBIIPAS2L);
 365         arm_smmu_tlb_sync_context(cookie);
 366 }
 367
 368 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
 369                                      unsigned long iova, size_t granule,
 370                                      void *cookie)
 371 {
 372         arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
 373                                   ARM_SMMU_CB_S2_TLBIIPAS2L);
 374 }
 375
 376 static void arm_smmu_tlb_inv_any_s2_v1(unsigned long iova, size_t size,
 377                                        size_t granule, void *cookie)
 378 {
 379         arm_smmu_tlb_inv_context_s2(cookie);
 380 }
 381 /*
 382  * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
 383  * almost negligible, but the benefit of getting the first one in as far ahead
 384  * of the sync as possible is significant, hence we don't just make this a
 385  * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
 386  * think.
 387  */
 388 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
 389                                         unsigned long iova, size_t granule,
 390                                         void *cookie)
 391 {
 392         struct arm_smmu_domain *smmu_domain = cookie;
 393         struct arm_smmu_device *smmu = smmu_domain->smmu;
 394
 395         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 396                 wmb();
 397
 398         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
 399 }
 400
 401 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
 402         .tlb_flush_all  = arm_smmu_tlb_inv_context_s1,
 403         .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
 404         .tlb_flush_leaf = arm_smmu_tlb_inv_leaf_s1,
 405         .tlb_add_page   = arm_smmu_tlb_add_page_s1,
 406 };
 407
 408 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
 409         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 410         .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
 411         .tlb_flush_leaf = arm_smmu_tlb_inv_leaf_s2,
 412         .tlb_add_page   = arm_smmu_tlb_add_page_s2,
 413 };
 414
 415 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
 416         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 417         .tlb_flush_walk = arm_smmu_tlb_inv_any_s2_v1,
 418         .tlb_flush_leaf = arm_smmu_tlb_inv_any_s2_v1,
 419         .tlb_add_page   = arm_smmu_tlb_add_page_s2_v1,
 420 };
 421
 422 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 423 {
 424         u32 fsr, fsynr, cbfrsynra;
 425         unsigned long iova;
 426         struct iommu_domain *domain = dev;
 427         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 428         struct arm_smmu_device *smmu = smmu_domain->smmu;
 429         int idx = smmu_domain->cfg.cbndx;
 430
 431         fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
 432         if (!(fsr & ARM_SMMU_FSR_FAULT))
 433                 return IRQ_NONE;
 434
 435         fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
 436         iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
 437         cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
 438
 439         dev_err_ratelimited(smmu->dev,
 440         "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
 441                             fsr, iova, fsynr, cbfrsynra, idx);
 442
 443         arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
 444         return IRQ_HANDLED;
 445 }
 446
 447 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 448 {
 449         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 450         struct arm_smmu_device *smmu = dev;
 451         static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
 452                                       DEFAULT_RATELIMIT_BURST);
 453
 454         gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
 455         gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
 456         gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
 457         gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
 458
 459         if (!gfsr)
 460                 return IRQ_NONE;
 461
 462         if (__ratelimit(&rs)) {
 463                 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
 464                     (gfsr & ARM_SMMU_sGFSR_USF))
 465                         dev_err(smmu->dev,
 466                                 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
 467                                 (u16)gfsynr1);
 468                 else
 469                         dev_err(smmu->dev,
 470                                 "Unexpected global fault, this could be serious\n");
 471                 dev_err(smmu->dev,
 472                         "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 473                         gfsr, gfsynr0, gfsynr1, gfsynr2);
 474         }
 475
 476         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
 477         return IRQ_HANDLED;
 478 }
 479
 480 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 481                                        struct io_pgtable_cfg *pgtbl_cfg)
 482 {
 483         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 484         struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
 485         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 486
 487         cb->cfg = cfg;
 488
 489         /* TCR */
 490         if (stage1) {
 491                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 492                         cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
 493                 } else {
 494                         cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
 495                         cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
 496                         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 497                                 cb->tcr[1] |= ARM_SMMU_TCR2_AS;
 498                         else
 499                                 cb->tcr[0] |= ARM_SMMU_TCR_EAE;
 500                 }
 501         } else {
 502                 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
 503         }
 504
 505         /* TTBRs */
 506         if (stage1) {
 507                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 508                         cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
 509                         cb->ttbr[1] = 0;
 510                 } else {
 511                         cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
 512                                                  cfg->asid);
 513                         cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
 514                                                  cfg->asid);
 515
 516                         if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
 517                                 cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
 518                         else
 519                                 cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
 520                 }
 521         } else {
 522                 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 523         }
 524
 525         /* MAIRs (stage-1 only) */
 526         if (stage1) {
 527                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 528                         cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
 529                         cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
 530                 } else {
 531                         cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
 532                         cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
 533                 }
 534         }
 535 }
 536
 537 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
 538 {
 539         u32 reg;
 540         bool stage1;
 541         struct arm_smmu_cb *cb = &smmu->cbs[idx];
 542         struct arm_smmu_cfg *cfg = cb->cfg;
 543
 544         /* Unassigned context banks only need disabling */
 545         if (!cfg) {
 546                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
 547                 return;
 548         }
 549
 550         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 551
 552         /* CBA2R */
 553         if (smmu->version > ARM_SMMU_V1) {
 554                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 555                         reg = ARM_SMMU_CBA2R_VA64;
 556                 else
 557                         reg = 0;
 558                 /* 16-bit VMIDs live in CBA2R */
 559                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 560                         reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
 561
 562                 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
 563         }
 564
 565         /* CBAR */
 566         reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
 567         if (smmu->version < ARM_SMMU_V2)
 568                 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
 569
 570         /*
 571          * Use the weakest shareability/memory types, so they are
 572          * overridden by the ttbcr/pte.
 573          */
 574         if (stage1) {
 575                 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
 576                                   ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
 577                        FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
 578                                   ARM_SMMU_CBAR_S1_MEMATTR_WB);
 579         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 580                 /* 8-bit VMIDs live in CBAR */
 581                 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
 582         }
 583         arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
 584
 585         /*
 586          * TCR
 587          * We must write this before the TTBRs, since it determines the
 588          * access behaviour of some fields (in particular, ASID[15:8]).
 589          */
 590         if (stage1 && smmu->version > ARM_SMMU_V1)
 591                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
 592         arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
 593
 594         /* TTBRs */
 595         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 596                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
 597                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
 598                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
 599         } else {
 600                 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
 601                 if (stage1)
 602                         arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
 603                                            cb->ttbr[1]);
 604         }
 605
 606         /* MAIRs (stage-1 only) */
 607         if (stage1) {
 608                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
 609                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
 610         }
 611
 612         /* SCTLR */
 613         reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
 614               ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
 615         if (stage1)
 616                 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
 617         if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
 618                 reg |= ARM_SMMU_SCTLR_E;
 619
 620         arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
 621 }
 622
 623 static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
 624                                        struct arm_smmu_device *smmu,
 625                                        struct device *dev, unsigned int start)
 626 {
 627         if (smmu->impl && smmu->impl->alloc_context_bank)
 628                 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start);
 629
 630         return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks);
 631 }
 632
 633 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 634                                         struct arm_smmu_device *smmu,
 635                                         struct device *dev)
 636 {
 637         int irq, start, ret = 0;
 638         unsigned long ias, oas;
 639         struct io_pgtable_ops *pgtbl_ops;
 640         struct io_pgtable_cfg pgtbl_cfg;
 641         enum io_pgtable_fmt fmt;
 642         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 643         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 644         irqreturn_t (*context_fault)(int irq, void *dev);
 645
 646         mutex_lock(&smmu_domain->init_mutex);
 647         if (smmu_domain->smmu)
 648                 goto out_unlock;
 649
 650         if (domain->type == IOMMU_DOMAIN_IDENTITY) {
 651                 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
 652                 smmu_domain->smmu = smmu;
 653                 goto out_unlock;
 654         }
 655
 656         /*
 657          * Mapping the requested stage onto what we support is surprisingly
 658          * complicated, mainly because the spec allows S1+S2 SMMUs without
 659          * support for nested translation. That means we end up with the
 660          * following table:
 661          *
 662          * Requested        Supported        Actual
 663          *     S1               N              S1
 664          *     S1             S1+S2            S1
 665          *     S1               S2             S2
 666          *     S1               S1             S1
 667          *     N                N              N
 668          *     N              S1+S2            S2
 669          *     N                S2             S2
 670          *     N                S1             S1
 671          *
 672          * Note that you can't actually request stage-2 mappings.
 673          */
 674         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 675                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 676         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 677                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 678
 679         /*
 680          * Choosing a suitable context format is even more fiddly. Until we
 681          * grow some way for the caller to express a preference, and/or move
 682          * the decision into the io-pgtable code where it arguably belongs,
 683          * just aim for the closest thing to the rest of the system, and hope
 684          * that the hardware isn't esoteric enough that we can't assume AArch64
 685          * support to be a superset of AArch32 support...
 686          */
 687         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 688                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 689         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
 690             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
 691             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
 692             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
 693                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 694         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 695             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 696                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 697                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 698                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 699
 700         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 701                 ret = -EINVAL;
 702                 goto out_unlock;
 703         }
 704
 705         switch (smmu_domain->stage) {
 706         case ARM_SMMU_DOMAIN_S1:
 707                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 708                 start = smmu->num_s2_context_banks;
 709                 ias = smmu->va_size;
 710                 oas = smmu->ipa_size;
 711                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 712                         fmt = ARM_64_LPAE_S1;
 713                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 714                         fmt = ARM_32_LPAE_S1;
 715                         ias = min(ias, 32UL);
 716                         oas = min(oas, 40UL);
 717                 } else {
 718                         fmt = ARM_V7S;
 719                         ias = min(ias, 32UL);
 720                         oas = min(oas, 32UL);
 721                 }
 722                 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
 723                 break;
 724         case ARM_SMMU_DOMAIN_NESTED:
 725                 /*
 726                  * We will likely want to change this if/when KVM gets
 727                  * involved.
 728                  */
 729         case ARM_SMMU_DOMAIN_S2:
 730                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 731                 start = 0;
 732                 ias = smmu->ipa_size;
 733                 oas = smmu->pa_size;
 734                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 735                         fmt = ARM_64_LPAE_S2;
 736                 } else {
 737                         fmt = ARM_32_LPAE_S2;
 738                         ias = min(ias, 40UL);
 739                         oas = min(oas, 40UL);
 740                 }
 741                 if (smmu->version == ARM_SMMU_V2)
 742                         smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
 743                 else
 744                         smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
 745                 break;
 746         default:
 747                 ret = -EINVAL;
 748                 goto out_unlock;
 749         }
 750
 751         ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start);
 752         if (ret < 0) {
 753                 goto out_unlock;
 754         }
 755
 756         smmu_domain->smmu = smmu;
 757
 758         cfg->cbndx = ret;
 759         if (smmu->version < ARM_SMMU_V2) {
 760                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 761                 cfg->irptndx %= smmu->num_context_irqs;
 762         } else {
 763                 cfg->irptndx = cfg->cbndx;
 764         }
 765
 766         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
 767                 cfg->vmid = cfg->cbndx + 1;
 768         else
 769                 cfg->asid = cfg->cbndx;
 770
 771         pgtbl_cfg = (struct io_pgtable_cfg) {
 772                 .pgsize_bitmap  = smmu->pgsize_bitmap,
 773                 .ias            = ias,
 774                 .oas            = oas,
 775                 .coherent_walk  = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
 776                 .tlb            = smmu_domain->flush_ops,
 777                 .iommu_dev      = smmu->dev,
 778         };
 779
 780         if (smmu->impl && smmu->impl->init_context) {
 781                 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev);
 782                 if (ret)
 783                         goto out_clear_smmu;
 784         }
 785
 786         if (smmu_domain->non_strict)
 787                 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
 788
 789         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 790         if (!pgtbl_ops) {
 791                 ret = -ENOMEM;
 792                 goto out_clear_smmu;
 793         }
 794
 795         /* Update the domain's page sizes to reflect the page table format */
 796         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 797
 798         if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
 799                 domain->geometry.aperture_start = ~0UL << ias;
 800                 domain->geometry.aperture_end = ~0UL;
 801         } else {
 802                 domain->geometry.aperture_end = (1UL << ias) - 1;
 803         }
 804
 805         domain->geometry.force_aperture = true;
 806
 807         /* Initialise the context bank with our page table cfg */
 808         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 809         arm_smmu_write_context_bank(smmu, cfg->cbndx);
 810
 811         /*
 812          * Request context fault interrupt. Do this last to avoid the
 813          * handler seeing a half-initialised domain state.
 814          */
 815         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 816
 817         if (smmu->impl && smmu->impl->context_fault)
 818                 context_fault = smmu->impl->context_fault;
 819         else
 820                 context_fault = arm_smmu_context_fault;
 821
 822         ret = devm_request_irq(smmu->dev, irq, context_fault,
 823                                IRQF_SHARED, "arm-smmu-context-fault", domain);
 824         if (ret < 0) {
 825                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 826                         cfg->irptndx, irq);
 827                 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
 828         }
 829
 830         mutex_unlock(&smmu_domain->init_mutex);
 831
 832         /* Publish page table ops for map/unmap */
 833         smmu_domain->pgtbl_ops = pgtbl_ops;
 834         return 0;
 835
 836 out_clear_smmu:
 837         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
 838         smmu_domain->smmu = NULL;
 839 out_unlock:
 840         mutex_unlock(&smmu_domain->init_mutex);
 841         return ret;
 842 }
 843
 844 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 845 {
 846         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 847         struct arm_smmu_device *smmu = smmu_domain->smmu;
 848         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 849         int ret, irq;
 850
 851         if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
 852                 return;
 853
 854         ret = arm_smmu_rpm_get(smmu);
 855         if (ret < 0)
 856                 return;
 857
 858         /*
 859          * Disable the context bank and free the page tables before freeing
 860          * it.
 861          */
 862         smmu->cbs[cfg->cbndx].cfg = NULL;
 863         arm_smmu_write_context_bank(smmu, cfg->cbndx);
 864
 865         if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
 866                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 867                 devm_free_irq(smmu->dev, irq, domain);
 868         }
 869
 870         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
 871         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
 872
 873         arm_smmu_rpm_put(smmu);
 874 }
 875
 876 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
 877 {
 878         struct arm_smmu_domain *smmu_domain;
 879
 880         if (type != IOMMU_DOMAIN_UNMANAGED &&
 881             type != IOMMU_DOMAIN_DMA &&
 882             type != IOMMU_DOMAIN_IDENTITY)
 883                 return NULL;
 884         /*
 885          * Allocate the domain and initialise some of its data structures.
 886          * We can't really do anything meaningful until we've added a
 887          * master.
 888          */
 889         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
 890         if (!smmu_domain)
 891                 return NULL;
 892
 893         if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
 894             iommu_get_dma_cookie(&smmu_domain->domain))) {
 895                 kfree(smmu_domain);
 896                 return NULL;
 897         }
 898
 899         mutex_init(&smmu_domain->init_mutex);
 900         spin_lock_init(&smmu_domain->cb_lock);
 901
 902         return &smmu_domain->domain;
 903 }
 904
 905 static void arm_smmu_domain_free(struct iommu_domain *domain)
 906 {
 907         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 908
 909         /*
 910          * Free the domain resources. We assume that all devices have
 911          * already been detached.
 912          */
 913         iommu_put_dma_cookie(domain);
 914         arm_smmu_destroy_domain_context(domain);
 915         kfree(smmu_domain);
 916 }
 917
 918 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
 919 {
 920         struct arm_smmu_smr *smr = smmu->smrs + idx;
 921         u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
 922                   FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
 923
 924         if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
 925                 reg |= ARM_SMMU_SMR_VALID;
 926         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
 927 }
 928
 929 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
 930 {
 931         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
 932         u32 reg;
 933
 934         if (smmu->impl && smmu->impl->write_s2cr) {
 935                 smmu->impl->write_s2cr(smmu, idx);
 936                 return;
 937         }
 938
 939         reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
 940               FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
 941               FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
 942
 943         if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
 944             smmu->smrs[idx].valid)
 945                 reg |= ARM_SMMU_S2CR_EXIDVALID;
 946         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
 947 }
 948
 949 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
 950 {
 951         arm_smmu_write_s2cr(smmu, idx);
 952         if (smmu->smrs)
 953                 arm_smmu_write_smr(smmu, idx);
 954 }
 955
 956 /*
 957  * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
 958  * should be called after sCR0 is written.
 959  */
 960 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
 961 {
 962         u32 smr;
 963         int i;
 964
 965         if (!smmu->smrs)
 966                 return;
 967         /*
 968          * If we've had to accommodate firmware memory regions, we may
 969          * have live SMRs by now; tread carefully...
 970          *
 971          * Somewhat perversely, not having a free SMR for this test implies we
 972          * can get away without it anyway, as we'll only be able to 'allocate'
 973          * these SMRs for the ID/mask values we're already trusting to be OK.
 974          */
 975         for (i = 0; i < smmu->num_mapping_groups; i++)
 976                 if (!smmu->smrs[i].valid)
 977                         goto smr_ok;
 978         return;
 979 smr_ok:
 980         /*
 981          * SMR.ID bits may not be preserved if the corresponding MASK
 982          * bits are set, so check each one separately. We can reject
 983          * masters later if they try to claim IDs outside these masks.
 984          */
 985         smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
 986         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
 987         smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
 988         smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
 989
 990         smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
 991         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
 992         smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
 993         smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
 994 }
 995
 996 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
 997 {
 998         struct arm_smmu_smr *smrs = smmu->smrs;
 999         int i, free_idx = -ENOSPC;
1000
1001         /* Stream indexing is blissfully easy */
1002         if (!smrs)
1003                 return id;
1004
1005         /* Validating SMRs is... less so */
1006         for (i = 0; i < smmu->num_mapping_groups; ++i) {
1007                 if (!smrs[i].valid) {
1008                         /*
1009                          * Note the first free entry we come across, which
1010                          * we'll claim in the end if nothing else matches.
1011                          */
1012                         if (free_idx < 0)
1013                                 free_idx = i;
1014                         continue;
1015                 }
1016                 /*
1017                  * If the new entry is _entirely_ matched by an existing entry,
1018                  * then reuse that, with the guarantee that there also cannot
1019                  * be any subsequent conflicting entries. In normal use we'd
1020                  * expect simply identical entries for this case, but there's
1021                  * no harm in accommodating the generalisation.
1022                  */
1023                 if ((mask & smrs[i].mask) == mask &&
1024                     !((id ^ smrs[i].id) & ~smrs[i].mask))
1025                         return i;
1026                 /*
1027                  * If the new entry has any other overlap with an existing one,
1028                  * though, then there always exists at least one stream ID
1029                  * which would cause a conflict, and we can't allow that risk.
1030                  */
1031                 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1032                         return -EINVAL;
1033         }
1034
1035         return free_idx;
1036 }
1037
1038 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1039 {
1040         if (--smmu->s2crs[idx].count)
1041                 return false;
1042
1043         smmu->s2crs[idx] = s2cr_init_val;
1044         if (smmu->smrs)
1045                 smmu->smrs[idx].valid = false;
1046
1047         return true;
1048 }
1049
1050 static int arm_smmu_master_alloc_smes(struct device *dev)
1051 {
1052         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1053         struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1054         struct arm_smmu_device *smmu = cfg->smmu;
1055         struct arm_smmu_smr *smrs = smmu->smrs;
1056         int i, idx, ret;
1057
1058         mutex_lock(&smmu->stream_map_mutex);
1059         /* Figure out a viable stream map entry allocation */
1060         for_each_cfg_sme(cfg, fwspec, i, idx) {
1061                 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1062                 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1063
1064                 if (idx != INVALID_SMENDX) {
1065                         ret = -EEXIST;
1066                         goto out_err;
1067                 }
1068
1069                 ret = arm_smmu_find_sme(smmu, sid, mask);
1070                 if (ret < 0)
1071                         goto out_err;
1072
1073                 idx = ret;
1074                 if (smrs && smmu->s2crs[idx].count == 0) {
1075                         smrs[idx].id = sid;
1076                         smrs[idx].mask = mask;
1077                         smrs[idx].valid = true;
1078                 }
1079                 smmu->s2crs[idx].count++;
1080                 cfg->smendx[i] = (s16)idx;
1081         }
1082
1083         /* It worked! Now, poke the actual hardware */
1084         for_each_cfg_sme(cfg, fwspec, i, idx)
1085                 arm_smmu_write_sme(smmu, idx);
1086
1087         mutex_unlock(&smmu->stream_map_mutex);
1088         return 0;
1089
1090 out_err:
1091         while (i--) {
1092                 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1093                 cfg->smendx[i] = INVALID_SMENDX;
1094         }
1095         mutex_unlock(&smmu->stream_map_mutex);
1096         return ret;
1097 }
1098
1099 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg,
1100                                       struct iommu_fwspec *fwspec)
1101 {
1102         struct arm_smmu_device *smmu = cfg->smmu;
1103         int i, idx;
1104
1105         mutex_lock(&smmu->stream_map_mutex);
1106         for_each_cfg_sme(cfg, fwspec, i, idx) {
1107                 if (arm_smmu_free_sme(smmu, idx))
1108                         arm_smmu_write_sme(smmu, idx);
1109                 cfg->smendx[i] = INVALID_SMENDX;
1110         }
1111         mutex_unlock(&smmu->stream_map_mutex);
1112 }
1113
1114 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1115                                       struct arm_smmu_master_cfg *cfg,
1116                                       struct iommu_fwspec *fwspec)
1117 {
1118         struct arm_smmu_device *smmu = smmu_domain->smmu;
1119         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1120         u8 cbndx = smmu_domain->cfg.cbndx;
1121         enum arm_smmu_s2cr_type type;
1122         int i, idx;
1123
1124         if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1125                 type = S2CR_TYPE_BYPASS;
1126         else
1127                 type = S2CR_TYPE_TRANS;
1128
1129         for_each_cfg_sme(cfg, fwspec, i, idx) {
1130                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1131                         continue;
1132
1133                 s2cr[idx].type = type;
1134                 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1135                 s2cr[idx].cbndx = cbndx;
1136                 arm_smmu_write_s2cr(smmu, idx);
1137         }
1138         return 0;
1139 }
1140
1141 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1142 {
1143         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1144         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1145         struct arm_smmu_master_cfg *cfg;
1146         struct arm_smmu_device *smmu;
1147         int ret;
1148
1149         if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1150                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1151                 return -ENXIO;
1152         }
1153
1154         /*
1155          * FIXME: The arch/arm DMA API code tries to attach devices to its own
1156          * domains between of_xlate() and probe_device() - we have no way to cope
1157          * with that, so until ARM gets converted to rely on groups and default
1158          * domains, just say no (but more politely than by dereferencing NULL).
1159          * This should be at least a WARN_ON once that's sorted.
1160          */
1161         cfg = dev_iommu_priv_get(dev);
1162         if (!cfg)
1163                 return -ENODEV;
1164
1165         smmu = cfg->smmu;
1166
1167         ret = arm_smmu_rpm_get(smmu);
1168         if (ret < 0)
1169                 return ret;
1170
1171         /* Ensure that the domain is finalised */
1172         ret = arm_smmu_init_domain_context(domain, smmu, dev);
1173         if (ret < 0)
1174                 goto rpm_put;
1175
1176         /*
1177          * Sanity check the domain. We don't support domains across
1178          * different SMMUs.
1179          */
1180         if (smmu_domain->smmu != smmu) {
1181                 dev_err(dev,
1182                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1183                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1184                 ret = -EINVAL;
1185                 goto rpm_put;
1186         }
1187
1188         /* Looks ok, so add the device to the domain */
1189         ret = arm_smmu_domain_add_master(smmu_domain, cfg, fwspec);
1190
1191         /*
1192          * Setup an autosuspend delay to avoid bouncing runpm state.
1193          * Otherwise, if a driver for a suspended consumer device
1194          * unmaps buffers, it will runpm resume/suspend for each one.
1195          *
1196          * For example, when used by a GPU device, when an application
1197          * or game exits, it can trigger unmapping 100s or 1000s of
1198          * buffers.  With a runpm cycle for each buffer, that adds up
1199          * to 5-10sec worth of reprogramming the context bank, while
1200          * the system appears to be locked up to the user.
1201          */
1202         pm_runtime_set_autosuspend_delay(smmu->dev, 20);
1203         pm_runtime_use_autosuspend(smmu->dev);
1204
1205 rpm_put:
1206         arm_smmu_rpm_put(smmu);
1207         return ret;
1208 }
1209
1210 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1211                         phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
1212 {
1213         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1214         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1215         int ret;
1216
1217         if (!ops)
1218                 return -ENODEV;
1219
1220         arm_smmu_rpm_get(smmu);
1221         ret = ops->map(ops, iova, paddr, size, prot, gfp);
1222         arm_smmu_rpm_put(smmu);
1223
1224         return ret;
1225 }
1226
1227 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1228                              size_t size, struct iommu_iotlb_gather *gather)
1229 {
1230         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1231         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1232         size_t ret;
1233
1234         if (!ops)
1235                 return 0;
1236
1237         arm_smmu_rpm_get(smmu);
1238         ret = ops->unmap(ops, iova, size, gather);
1239         arm_smmu_rpm_put(smmu);
1240
1241         return ret;
1242 }
1243
1244 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1245 {
1246         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1247         struct arm_smmu_device *smmu = smmu_domain->smmu;
1248
1249         if (smmu_domain->flush_ops) {
1250                 arm_smmu_rpm_get(smmu);
1251                 smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1252                 arm_smmu_rpm_put(smmu);
1253         }
1254 }
1255
1256 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1257                                 struct iommu_iotlb_gather *gather)
1258 {
1259         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1260         struct arm_smmu_device *smmu = smmu_domain->smmu;
1261
1262         if (!smmu)
1263                 return;
1264
1265         arm_smmu_rpm_get(smmu);
1266         if (smmu->version == ARM_SMMU_V2 ||
1267             smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1268                 arm_smmu_tlb_sync_context(smmu_domain);
1269         else
1270                 arm_smmu_tlb_sync_global(smmu);
1271         arm_smmu_rpm_put(smmu);
1272 }
1273
1274 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1275                                               dma_addr_t iova)
1276 {
1277         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1278         struct arm_smmu_device *smmu = smmu_domain->smmu;
1279         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1280         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1281         struct device *dev = smmu->dev;
1282         void __iomem *reg;
1283         u32 tmp;
1284         u64 phys;
1285         unsigned long va, flags;
1286         int ret, idx = cfg->cbndx;
1287         phys_addr_t addr = 0;
1288
1289         ret = arm_smmu_rpm_get(smmu);
1290         if (ret < 0)
1291                 return 0;
1292
1293         spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1294         va = iova & ~0xfffUL;
1295         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1296                 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1297         else
1298                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1299
1300         reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1301         if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_ATSR_ACTIVE),
1302                                       5, 50)) {
1303                 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1304                 dev_err(dev,
1305                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1306                         &iova);
1307                 arm_smmu_rpm_put(smmu);
1308                 return ops->iova_to_phys(ops, iova);
1309         }
1310
1311         phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1312         spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1313         if (phys & ARM_SMMU_CB_PAR_F) {
1314                 dev_err(dev, "translation fault!\n");
1315                 dev_err(dev, "PAR = 0x%llx\n", phys);
1316                 goto out;
1317         }
1318
1319         addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1320 out:
1321         arm_smmu_rpm_put(smmu);
1322
1323         return addr;
1324 }
1325
1326 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1327                                         dma_addr_t iova)
1328 {
1329         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1330         struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1331
1332         if (domain->type == IOMMU_DOMAIN_IDENTITY)
1333                 return iova;
1334
1335         if (!ops)
1336                 return 0;
1337
1338         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1339                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1340                 return arm_smmu_iova_to_phys_hard(domain, iova);
1341
1342         return ops->iova_to_phys(ops, iova);
1343 }
1344
1345 static bool arm_smmu_capable(enum iommu_cap cap)
1346 {
1347         switch (cap) {
1348         case IOMMU_CAP_CACHE_COHERENCY:
1349                 /*
1350                  * Return true here as the SMMU can always send out coherent
1351                  * requests.
1352                  */
1353                 return true;
1354         case IOMMU_CAP_NOEXEC:
1355                 return true;
1356         default:
1357                 return false;
1358         }
1359 }
1360
1361 static
1362 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1363 {
1364         struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
1365                                                           fwnode);
1366         put_device(dev);
1367         return dev ? dev_get_drvdata(dev) : NULL;
1368 }
1369
1370 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
1371 {
1372         struct arm_smmu_device *smmu = NULL;
1373         struct arm_smmu_master_cfg *cfg;
1374         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1375         int i, ret;
1376
1377         if (using_legacy_binding) {
1378                 ret = arm_smmu_register_legacy_master(dev, &smmu);
1379
1380                 /*
1381                  * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1382                  * will allocate/initialise a new one. Thus we need to update fwspec for
1383                  * later use.
1384                  */
1385                 fwspec = dev_iommu_fwspec_get(dev);
1386                 if (ret)
1387                         goto out_free;
1388         } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1389                 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1390         } else {
1391                 return ERR_PTR(-ENODEV);
1392         }
1393
1394         ret = -EINVAL;
1395         for (i = 0; i < fwspec->num_ids; i++) {
1396                 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1397                 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1398
1399                 if (sid & ~smmu->streamid_mask) {
1400                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1401                                 sid, smmu->streamid_mask);
1402                         goto out_free;
1403                 }
1404                 if (mask & ~smmu->smr_mask_mask) {
1405                         dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1406                                 mask, smmu->smr_mask_mask);
1407                         goto out_free;
1408                 }
1409         }
1410
1411         ret = -ENOMEM;
1412         cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1413                       GFP_KERNEL);
1414         if (!cfg)
1415                 goto out_free;
1416
1417         cfg->smmu = smmu;
1418         dev_iommu_priv_set(dev, cfg);
1419         while (i--)
1420                 cfg->smendx[i] = INVALID_SMENDX;
1421
1422         ret = arm_smmu_rpm_get(smmu);
1423         if (ret < 0)
1424                 goto out_cfg_free;
1425
1426         ret = arm_smmu_master_alloc_smes(dev);
1427         arm_smmu_rpm_put(smmu);
1428
1429         if (ret)
1430                 goto out_cfg_free;
1431
1432         device_link_add(dev, smmu->dev,
1433                         DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1434
1435         return &smmu->iommu;
1436
1437 out_cfg_free:
1438         kfree(cfg);
1439 out_free:
1440         iommu_fwspec_free(dev);
1441         return ERR_PTR(ret);
1442 }
1443
1444 static void arm_smmu_release_device(struct device *dev)
1445 {
1446         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1447         struct arm_smmu_master_cfg *cfg;
1448         struct arm_smmu_device *smmu;
1449         int ret;
1450
1451         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1452                 return;
1453
1454         cfg  = dev_iommu_priv_get(dev);
1455         smmu = cfg->smmu;
1456
1457         ret = arm_smmu_rpm_get(smmu);
1458         if (ret < 0)
1459                 return;
1460
1461         arm_smmu_master_free_smes(cfg, fwspec);
1462
1463         arm_smmu_rpm_put(smmu);
1464
1465         dev_iommu_priv_set(dev, NULL);
1466         kfree(cfg);
1467         iommu_fwspec_free(dev);
1468 }
1469
1470 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1471 {
1472         struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1473         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1474         struct arm_smmu_device *smmu = cfg->smmu;
1475         struct iommu_group *group = NULL;
1476         int i, idx;
1477
1478         for_each_cfg_sme(cfg, fwspec, i, idx) {
1479                 if (group && smmu->s2crs[idx].group &&
1480                     group != smmu->s2crs[idx].group)
1481                         return ERR_PTR(-EINVAL);
1482
1483                 group = smmu->s2crs[idx].group;
1484         }
1485
1486         if (group)
1487                 return iommu_group_ref_get(group);
1488
1489         if (dev_is_pci(dev))
1490                 group = pci_device_group(dev);
1491         else if (dev_is_fsl_mc(dev))
1492                 group = fsl_mc_device_group(dev);
1493         else
1494                 group = generic_device_group(dev);
1495
1496         /* Remember group for faster lookups */
1497         if (!IS_ERR(group))
1498                 for_each_cfg_sme(cfg, fwspec, i, idx)
1499                         smmu->s2crs[idx].group = group;
1500
1501         return group;
1502 }
1503
1504 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1505                                     enum iommu_attr attr, void *data)
1506 {
1507         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1508
1509         switch(domain->type) {
1510         case IOMMU_DOMAIN_UNMANAGED:
1511                 switch (attr) {
1512                 case DOMAIN_ATTR_NESTING:
1513                         *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1514                         return 0;
1515                 default:
1516                         return -ENODEV;
1517                 }
1518                 break;
1519         case IOMMU_DOMAIN_DMA:
1520                 switch (attr) {
1521                 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1522                         *(int *)data = smmu_domain->non_strict;
1523                         return 0;
1524                 default:
1525                         return -ENODEV;
1526                 }
1527                 break;
1528         default:
1529                 return -EINVAL;
1530         }
1531 }
1532
1533 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1534                                     enum iommu_attr attr, void *data)
1535 {
1536         int ret = 0;
1537         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1538
1539         mutex_lock(&smmu_domain->init_mutex);
1540
1541         switch(domain->type) {
1542         case IOMMU_DOMAIN_UNMANAGED:
1543                 switch (attr) {
1544                 case DOMAIN_ATTR_NESTING:
1545                         if (smmu_domain->smmu) {
1546                                 ret = -EPERM;
1547                                 goto out_unlock;
1548                         }
1549
1550                         if (*(int *)data)
1551                                 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1552                         else
1553                                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1554                         break;
1555                 default:
1556                         ret = -ENODEV;
1557                 }
1558                 break;
1559         case IOMMU_DOMAIN_DMA:
1560                 switch (attr) {
1561                 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1562                         smmu_domain->non_strict = *(int *)data;
1563                         break;
1564                 default:
1565                         ret = -ENODEV;
1566                 }
1567                 break;
1568         default:
1569                 ret = -EINVAL;
1570         }
1571 out_unlock:
1572         mutex_unlock(&smmu_domain->init_mutex);
1573         return ret;
1574 }
1575
1576 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1577 {
1578         u32 mask, fwid = 0;
1579
1580         if (args->args_count > 0)
1581                 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1582
1583         if (args->args_count > 1)
1584                 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1585         else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1586                 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1587
1588         return iommu_fwspec_add_ids(dev, &fwid, 1);
1589 }
1590
1591 static void arm_smmu_get_resv_regions(struct device *dev,
1592                                       struct list_head *head)
1593 {
1594         struct iommu_resv_region *region;
1595         int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1596
1597         region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1598                                          prot, IOMMU_RESV_SW_MSI);
1599         if (!region)
1600                 return;
1601
1602         list_add_tail(&region->list, head);
1603
1604         iommu_dma_get_resv_regions(dev, head);
1605 }
1606
1607 static int arm_smmu_def_domain_type(struct device *dev)
1608 {
1609         struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1610         const struct arm_smmu_impl *impl = cfg->smmu->impl;
1611
1612         if (impl && impl->def_domain_type)
1613                 return impl->def_domain_type(dev);
1614
1615         return 0;
1616 }
1617
1618 static struct iommu_ops arm_smmu_ops = {
1619         .capable                = arm_smmu_capable,
1620         .domain_alloc           = arm_smmu_domain_alloc,
1621         .domain_free            = arm_smmu_domain_free,
1622         .attach_dev             = arm_smmu_attach_dev,
1623         .map                    = arm_smmu_map,
1624         .unmap                  = arm_smmu_unmap,
1625         .flush_iotlb_all        = arm_smmu_flush_iotlb_all,
1626         .iotlb_sync             = arm_smmu_iotlb_sync,
1627         .iova_to_phys           = arm_smmu_iova_to_phys,
1628         .probe_device           = arm_smmu_probe_device,
1629         .release_device         = arm_smmu_release_device,
1630         .device_group           = arm_smmu_device_group,
1631         .domain_get_attr        = arm_smmu_domain_get_attr,
1632         .domain_set_attr        = arm_smmu_domain_set_attr,
1633         .of_xlate               = arm_smmu_of_xlate,
1634         .get_resv_regions       = arm_smmu_get_resv_regions,
1635         .put_resv_regions       = generic_iommu_put_resv_regions,
1636         .def_domain_type        = arm_smmu_def_domain_type,
1637         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1638 };
1639
1640 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1641 {
1642         int i;
1643         u32 reg;
1644
1645         /* clear global FSR */
1646         reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1647         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1648
1649         /*
1650          * Reset stream mapping groups: Initial values mark all SMRn as
1651          * invalid and all S2CRn as bypass unless overridden.
1652          */
1653         for (i = 0; i < smmu->num_mapping_groups; ++i)
1654                 arm_smmu_write_sme(smmu, i);
1655
1656         /* Make sure all context banks are disabled and clear CB_FSR  */
1657         for (i = 0; i < smmu->num_context_banks; ++i) {
1658                 arm_smmu_write_context_bank(smmu, i);
1659                 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT);
1660         }
1661
1662         /* Invalidate the TLB, just in case */
1663         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1664         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1665
1666         reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1667
1668         /* Enable fault reporting */
1669         reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1670                 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1671
1672         /* Disable TLB broadcasting. */
1673         reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1674
1675         /* Enable client access, handling unmatched streams as appropriate */
1676         reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1677         if (disable_bypass)
1678                 reg |= ARM_SMMU_sCR0_USFCFG;
1679         else
1680                 reg &= ~ARM_SMMU_sCR0_USFCFG;
1681
1682         /* Disable forced broadcasting */
1683         reg &= ~ARM_SMMU_sCR0_FB;
1684
1685         /* Don't upgrade barriers */
1686         reg &= ~(ARM_SMMU_sCR0_BSU);
1687
1688         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1689                 reg |= ARM_SMMU_sCR0_VMID16EN;
1690
1691         if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1692                 reg |= ARM_SMMU_sCR0_EXIDENABLE;
1693
1694         if (smmu->impl && smmu->impl->reset)
1695                 smmu->impl->reset(smmu);
1696
1697         /* Push the button */
1698         arm_smmu_tlb_sync_global(smmu);
1699         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1700 }
1701
1702 static int arm_smmu_id_size_to_bits(int size)
1703 {
1704         switch (size) {
1705         case 0:
1706                 return 32;
1707         case 1:
1708                 return 36;
1709         case 2:
1710                 return 40;
1711         case 3:
1712                 return 42;
1713         case 4:
1714                 return 44;
1715         case 5:
1716         default:
1717                 return 48;
1718         }
1719 }
1720
1721 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1722 {
1723         unsigned int size;
1724         u32 id;
1725         bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1726         int i, ret;
1727
1728         dev_notice(smmu->dev, "probing hardware configuration...\n");
1729         dev_notice(smmu->dev, "SMMUv%d with:\n",
1730                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1731
1732         /* ID0 */
1733         id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1734
1735         /* Restrict available stages based on module parameter */
1736         if (force_stage == 1)
1737                 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1738         else if (force_stage == 2)
1739                 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1740
1741         if (id & ARM_SMMU_ID0_S1TS) {
1742                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1743                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1744         }
1745
1746         if (id & ARM_SMMU_ID0_S2TS) {
1747                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1748                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1749         }
1750
1751         if (id & ARM_SMMU_ID0_NTS) {
1752                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1753                 dev_notice(smmu->dev, "\tnested translation\n");
1754         }
1755
1756         if (!(smmu->features &
1757                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1758                 dev_err(smmu->dev, "\tno translation support!\n");
1759                 return -ENODEV;
1760         }
1761
1762         if ((id & ARM_SMMU_ID0_S1TS) &&
1763             ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1764                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1765                 dev_notice(smmu->dev, "\taddress translation ops\n");
1766         }
1767
1768         /*
1769          * In order for DMA API calls to work properly, we must defer to what
1770          * the FW says about coherency, regardless of what the hardware claims.
1771          * Fortunately, this also opens up a workaround for systems where the
1772          * ID register value has ended up configured incorrectly.
1773          */
1774         cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1775         if (cttw_fw || cttw_reg)
1776                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1777                            cttw_fw ? "" : "non-");
1778         if (cttw_fw != cttw_reg)
1779                 dev_notice(smmu->dev,
1780                            "\t(IDR0.CTTW overridden by FW configuration)\n");
1781
1782         /* Max. number of entries we have for stream matching/indexing */
1783         if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1784                 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1785                 size = 1 << 16;
1786         } else {
1787                 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1788         }
1789         smmu->streamid_mask = size - 1;
1790         if (id & ARM_SMMU_ID0_SMS) {
1791                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1792                 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1793                 if (size == 0) {
1794                         dev_err(smmu->dev,
1795                                 "stream-matching supported, but no SMRs present!\n");
1796                         return -ENODEV;
1797                 }
1798
1799                 /* Zero-initialised to mark as invalid */
1800                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1801                                           GFP_KERNEL);
1802                 if (!smmu->smrs)
1803                         return -ENOMEM;
1804
1805                 dev_notice(smmu->dev,
1806                            "\tstream matching with %u register groups", size);
1807         }
1808         /* s2cr->type == 0 means translation, so initialise explicitly */
1809         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1810                                          GFP_KERNEL);
1811         if (!smmu->s2crs)
1812                 return -ENOMEM;
1813         for (i = 0; i < size; i++)
1814                 smmu->s2crs[i] = s2cr_init_val;
1815
1816         smmu->num_mapping_groups = size;
1817         mutex_init(&smmu->stream_map_mutex);
1818         spin_lock_init(&smmu->global_sync_lock);
1819
1820         if (smmu->version < ARM_SMMU_V2 ||
1821             !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1822                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1823                 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1824                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1825         }
1826
1827         /* ID1 */
1828         id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1829         smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1830
1831         /* Check for size mismatch of SMMU address space from mapped region */
1832         size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1833         if (smmu->numpage != 2 * size << smmu->pgshift)
1834                 dev_warn(smmu->dev,
1835                         "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1836                         2 * size << smmu->pgshift, smmu->numpage);
1837         /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1838         smmu->numpage = size;
1839
1840         smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1841         smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1842         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1843                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1844                 return -ENODEV;
1845         }
1846         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1847                    smmu->num_context_banks, smmu->num_s2_context_banks);
1848         smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1849                                  sizeof(*smmu->cbs), GFP_KERNEL);
1850         if (!smmu->cbs)
1851                 return -ENOMEM;
1852
1853         /* ID2 */
1854         id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1855         size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1856         smmu->ipa_size = size;
1857
1858         /* The output mask is also applied for bypass */
1859         size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1860         smmu->pa_size = size;
1861
1862         if (id & ARM_SMMU_ID2_VMID16)
1863                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1864
1865         /*
1866          * What the page table walker can address actually depends on which
1867          * descriptor format is in use, but since a) we don't know that yet,
1868          * and b) it can vary per context bank, this will have to do...
1869          */
1870         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1871                 dev_warn(smmu->dev,
1872                          "failed to set DMA mask for table walker\n");
1873
1874         if (smmu->version < ARM_SMMU_V2) {
1875                 smmu->va_size = smmu->ipa_size;
1876                 if (smmu->version == ARM_SMMU_V1_64K)
1877                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1878         } else {
1879                 size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1880                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1881                 if (id & ARM_SMMU_ID2_PTFS_4K)
1882                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1883                 if (id & ARM_SMMU_ID2_PTFS_16K)
1884                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1885                 if (id & ARM_SMMU_ID2_PTFS_64K)
1886                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1887         }
1888
1889         if (smmu->impl && smmu->impl->cfg_probe) {
1890                 ret = smmu->impl->cfg_probe(smmu);
1891                 if (ret)
1892                         return ret;
1893         }
1894
1895         /* Now we've corralled the various formats, what'll it do? */
1896         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1897                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1898         if (smmu->features &
1899             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1900                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1901         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1902                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1903         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1904                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1905
1906         if (arm_smmu_ops.pgsize_bitmap == -1UL)
1907                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1908         else
1909                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1910         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1911                    smmu->pgsize_bitmap);
1912
1913
1914         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1915                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1916                            smmu->va_size, smmu->ipa_size);
1917
1918         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1919                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1920                            smmu->ipa_size, smmu->pa_size);
1921
1922         return 0;
1923 }
1924
1925 struct arm_smmu_match_data {
1926         enum arm_smmu_arch_version version;
1927         enum arm_smmu_implementation model;
1928 };
1929
1930 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
1931 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1932
1933 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1934 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1935 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1936 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1937 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1938 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1939
1940 static const struct of_device_id arm_smmu_of_match[] = {
1941         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1942         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1943         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1944         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1945         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1946         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1947         { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 },
1948         { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1949         { },
1950 };
1951 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1952
1953 #ifdef CONFIG_ACPI
1954 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1955 {
1956         int ret = 0;
1957
1958         switch (model) {
1959         case ACPI_IORT_SMMU_V1:
1960         case ACPI_IORT_SMMU_CORELINK_MMU400:
1961                 smmu->version = ARM_SMMU_V1;
1962                 smmu->model = GENERIC_SMMU;
1963                 break;
1964         case ACPI_IORT_SMMU_CORELINK_MMU401:
1965                 smmu->version = ARM_SMMU_V1_64K;
1966                 smmu->model = GENERIC_SMMU;
1967                 break;
1968         case ACPI_IORT_SMMU_V2:
1969                 smmu->version = ARM_SMMU_V2;
1970                 smmu->model = GENERIC_SMMU;
1971                 break;
1972         case ACPI_IORT_SMMU_CORELINK_MMU500:
1973                 smmu->version = ARM_SMMU_V2;
1974                 smmu->model = ARM_MMU500;
1975                 break;
1976         case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1977                 smmu->version = ARM_SMMU_V2;
1978                 smmu->model = CAVIUM_SMMUV2;
1979                 break;
1980         default:
1981                 ret = -ENODEV;
1982         }
1983
1984         return ret;
1985 }
1986
1987 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1988                                       struct arm_smmu_device *smmu)
1989 {
1990         struct device *dev = smmu->dev;
1991         struct acpi_iort_node *node =
1992                 *(struct acpi_iort_node **)dev_get_platdata(dev);
1993         struct acpi_iort_smmu *iort_smmu;
1994         int ret;
1995
1996         /* Retrieve SMMU1/2 specific data */
1997         iort_smmu = (struct acpi_iort_smmu *)node->node_data;
1998
1999         ret = acpi_smmu_get_data(iort_smmu->model, smmu);
2000         if (ret < 0)
2001                 return ret;
2002
2003         /* Ignore the configuration access interrupt */
2004         smmu->num_global_irqs = 1;
2005
2006         if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2007                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2008
2009         return 0;
2010 }
2011 #else
2012 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2013                                              struct arm_smmu_device *smmu)
2014 {
2015         return -ENODEV;
2016 }
2017 #endif
2018
2019 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2020                                     struct arm_smmu_device *smmu)
2021 {
2022         const struct arm_smmu_match_data *data;
2023         struct device *dev = &pdev->dev;
2024         bool legacy_binding;
2025
2026         if (of_property_read_u32(dev->of_node, "#global-interrupts",
2027                                  &smmu->num_global_irqs)) {
2028                 dev_err(dev, "missing #global-interrupts property\n");
2029                 return -ENODEV;
2030         }
2031
2032         data = of_device_get_match_data(dev);
2033         smmu->version = data->version;
2034         smmu->model = data->model;
2035
2036         legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2037         if (legacy_binding && !using_generic_binding) {
2038                 if (!using_legacy_binding) {
2039                         pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
2040                                   IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
2041                 }
2042                 using_legacy_binding = true;
2043         } else if (!legacy_binding && !using_legacy_binding) {
2044                 using_generic_binding = true;
2045         } else {
2046                 dev_err(dev, "not probing due to mismatched DT properties\n");
2047                 return -ENODEV;
2048         }
2049
2050         if (of_dma_is_coherent(dev->of_node))
2051                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2052
2053         return 0;
2054 }
2055
2056 static int arm_smmu_bus_init(struct iommu_ops *ops)
2057 {
2058         int err;
2059
2060         /* Oh, for a proper bus abstraction */
2061         if (!iommu_present(&platform_bus_type)) {
2062                 err = bus_set_iommu(&platform_bus_type, ops);
2063                 if (err)
2064                         return err;
2065         }
2066 #ifdef CONFIG_ARM_AMBA
2067         if (!iommu_present(&amba_bustype)) {
2068                 err = bus_set_iommu(&amba_bustype, ops);
2069                 if (err)
2070                         goto err_reset_platform_ops;
2071         }
2072 #endif
2073 #ifdef CONFIG_PCI
2074         if (!iommu_present(&pci_bus_type)) {
2075                 err = bus_set_iommu(&pci_bus_type, ops);
2076                 if (err)
2077                         goto err_reset_amba_ops;
2078         }
2079 #endif
2080 #ifdef CONFIG_FSL_MC_BUS
2081         if (!iommu_present(&fsl_mc_bus_type)) {
2082                 err = bus_set_iommu(&fsl_mc_bus_type, ops);
2083                 if (err)
2084                         goto err_reset_pci_ops;
2085         }
2086 #endif
2087         return 0;
2088
2089 err_reset_pci_ops: __maybe_unused;
2090 #ifdef CONFIG_PCI
2091         bus_set_iommu(&pci_bus_type, NULL);
2092 #endif
2093 err_reset_amba_ops: __maybe_unused;
2094 #ifdef CONFIG_ARM_AMBA
2095         bus_set_iommu(&amba_bustype, NULL);
2096 #endif
2097 err_reset_platform_ops: __maybe_unused;
2098         bus_set_iommu(&platform_bus_type, NULL);
2099         return err;
2100 }
2101
2102 static int arm_smmu_device_probe(struct platform_device *pdev)
2103 {
2104         struct resource *res;
2105         resource_size_t ioaddr;
2106         struct arm_smmu_device *smmu;
2107         struct device *dev = &pdev->dev;
2108         int num_irqs, i, err;
2109         irqreturn_t (*global_fault)(int irq, void *dev);
2110
2111         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2112         if (!smmu) {
2113                 dev_err(dev, "failed to allocate arm_smmu_device\n");
2114                 return -ENOMEM;
2115         }
2116         smmu->dev = dev;
2117
2118         if (dev->of_node)
2119                 err = arm_smmu_device_dt_probe(pdev, smmu);
2120         else
2121                 err = arm_smmu_device_acpi_probe(pdev, smmu);
2122
2123         if (err)
2124                 return err;
2125
2126         smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
2127         if (IS_ERR(smmu->base))
2128                 return PTR_ERR(smmu->base);
2129         ioaddr = res->start;
2130         /*
2131          * The resource size should effectively match the value of SMMU_TOP;
2132          * stash that temporarily until we know PAGESIZE to validate it with.
2133          */
2134         smmu->numpage = resource_size(res);
2135
2136         smmu = arm_smmu_impl_init(smmu);
2137         if (IS_ERR(smmu))
2138                 return PTR_ERR(smmu);
2139
2140         num_irqs = 0;
2141         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2142                 num_irqs++;
2143                 if (num_irqs > smmu->num_global_irqs)
2144                         smmu->num_context_irqs++;
2145         }
2146
2147         if (!smmu->num_context_irqs) {
2148                 dev_err(dev, "found %d interrupts but expected at least %d\n",
2149                         num_irqs, smmu->num_global_irqs + 1);
2150                 return -ENODEV;
2151         }
2152
2153         smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),
2154                                   GFP_KERNEL);
2155         if (!smmu->irqs) {
2156                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2157                 return -ENOMEM;
2158         }
2159
2160         for (i = 0; i < num_irqs; ++i) {
2161                 int irq = platform_get_irq(pdev, i);
2162
2163                 if (irq < 0)
2164                         return -ENODEV;
2165                 smmu->irqs[i] = irq;
2166         }
2167
2168         err = devm_clk_bulk_get_all(dev, &smmu->clks);
2169         if (err < 0) {
2170                 dev_err(dev, "failed to get clocks %d\n", err);
2171                 return err;
2172         }
2173         smmu->num_clks = err;
2174
2175         err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2176         if (err)
2177                 return err;
2178
2179         err = arm_smmu_device_cfg_probe(smmu);
2180         if (err)
2181                 return err;
2182
2183         if (smmu->version == ARM_SMMU_V2) {
2184                 if (smmu->num_context_banks > smmu->num_context_irqs) {
2185                         dev_err(dev,
2186                               "found only %d context irq(s) but %d required\n",
2187                               smmu->num_context_irqs, smmu->num_context_banks);
2188                         return -ENODEV;
2189                 }
2190
2191                 /* Ignore superfluous interrupts */
2192                 smmu->num_context_irqs = smmu->num_context_banks;
2193         }
2194
2195         if (smmu->impl && smmu->impl->global_fault)
2196                 global_fault = smmu->impl->global_fault;
2197         else
2198                 global_fault = arm_smmu_global_fault;
2199
2200         for (i = 0; i < smmu->num_global_irqs; ++i) {
2201                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2202                                        global_fault,
2203                                        IRQF_SHARED,
2204                                        "arm-smmu global fault",
2205                                        smmu);
2206                 if (err) {
2207                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
2208                                 i, smmu->irqs[i]);
2209                         return err;
2210                 }
2211         }
2212
2213         err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2214                                      "smmu.%pa", &ioaddr);
2215         if (err) {
2216                 dev_err(dev, "Failed to register iommu in sysfs\n");
2217                 return err;
2218         }
2219
2220         iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2221         iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2222
2223         err = iommu_device_register(&smmu->iommu);
2224         if (err) {
2225                 dev_err(dev, "Failed to register iommu\n");
2226                 return err;
2227         }
2228
2229         platform_set_drvdata(pdev, smmu);
2230         arm_smmu_device_reset(smmu);
2231         arm_smmu_test_smr_masks(smmu);
2232
2233         /*
2234          * We want to avoid touching dev->power.lock in fastpaths unless
2235          * it's really going to do something useful - pm_runtime_enabled()
2236          * can serve as an ideal proxy for that decision. So, conditionally
2237          * enable pm_runtime.
2238          */
2239         if (dev->pm_domain) {
2240                 pm_runtime_set_active(dev);
2241                 pm_runtime_enable(dev);
2242         }
2243
2244         /*
2245          * For ACPI and generic DT bindings, an SMMU will be probed before
2246          * any device which might need it, so we want the bus ops in place
2247          * ready to handle default domain setup as soon as any SMMU exists.
2248          */
2249         if (!using_legacy_binding)
2250                 return arm_smmu_bus_init(&arm_smmu_ops);
2251
2252         return 0;
2253 }
2254
2255 static int arm_smmu_device_remove(struct platform_device *pdev)
2256 {
2257         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2258
2259         if (!smmu)
2260                 return -ENODEV;
2261
2262         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2263                 dev_notice(&pdev->dev, "disabling translation\n");
2264
2265         arm_smmu_bus_init(NULL);
2266         iommu_device_unregister(&smmu->iommu);
2267         iommu_device_sysfs_remove(&smmu->iommu);
2268
2269         arm_smmu_rpm_get(smmu);
2270         /* Turn the thing off */
2271         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2272         arm_smmu_rpm_put(smmu);
2273
2274         if (pm_runtime_enabled(smmu->dev))
2275                 pm_runtime_force_suspend(smmu->dev);
2276         else
2277                 clk_bulk_disable(smmu->num_clks, smmu->clks);
2278
2279         clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2280         return 0;
2281 }
2282
2283 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2284 {
2285         arm_smmu_device_remove(pdev);
2286 }
2287
2288 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2289 {
2290         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2291         int ret;
2292
2293         ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2294         if (ret)
2295                 return ret;
2296
2297         arm_smmu_device_reset(smmu);
2298
2299         return 0;
2300 }
2301
2302 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2303 {
2304         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2305
2306         clk_bulk_disable(smmu->num_clks, smmu->clks);
2307
2308         return 0;
2309 }
2310
2311 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2312 {
2313         if (pm_runtime_suspended(dev))
2314                 return 0;
2315
2316         return arm_smmu_runtime_resume(dev);
2317 }
2318
2319 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2320 {
2321         if (pm_runtime_suspended(dev))
2322                 return 0;
2323
2324         return arm_smmu_runtime_suspend(dev);
2325 }
2326
2327 static const struct dev_pm_ops arm_smmu_pm_ops = {
2328         SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2329         SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2330                            arm_smmu_runtime_resume, NULL)
2331 };
2332
2333 static struct platform_driver arm_smmu_driver = {
2334         .driver = {
2335                 .name                   = "arm-smmu",
2336                 .of_match_table         = arm_smmu_of_match,
2337                 .pm                     = &arm_smmu_pm_ops,
2338                 .suppress_bind_attrs    = true,
2339         },
2340         .probe  = arm_smmu_device_probe,
2341         .remove = arm_smmu_device_remove,
2342         .shutdown = arm_smmu_device_shutdown,
2343 };
2344 module_platform_driver(arm_smmu_driver);
2345
2346 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2347 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2348 MODULE_ALIAS("platform:arm-smmu");
2349 MODULE_LICENSE("GPL v2");