1 // SPDX-License-Identifier: GPL-2.0-only
3 * IOMMU API for ARM architected SMMU implementations.
5 * Copyright (C) 2013 ARM Limited
7 * Author: Will Deacon <will.deacon@arm.com>
9 * This driver currently supports:
10 * - SMMUv1 and v2 implementations
11 * - Stream-matching and stream-indexing
12 * - v7/v8 long-descriptor format
13 * - Non-secure access to the SMMU
14 * - Context fault reporting
15 * - Extended Stream ID (16 bit)
18 #define pr_fmt(fmt) "arm-smmu: " fmt
20 #include <linux/acpi.h>
21 #include <linux/acpi_iort.h>
22 #include <linux/bitfield.h>
23 #include <linux/delay.h>
24 #include <linux/dma-iommu.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/err.h>
27 #include <linux/interrupt.h>
29 #include <linux/iopoll.h>
30 #include <linux/init.h>
31 #include <linux/moduleparam.h>
33 #include <linux/of_address.h>
34 #include <linux/of_device.h>
35 #include <linux/of_iommu.h>
36 #include <linux/pci.h>
37 #include <linux/platform_device.h>
38 #include <linux/pm_runtime.h>
39 #include <linux/slab.h>
41 #include <linux/amba/bus.h>
42 #include <linux/fsl/mc.h>
47 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
48 * global register space are still, in fact, using a hypervisor to mediate it
49 * by trapping and emulating register accesses. Sadly, some deployed versions
50 * of said trapping code have bugs wherein they go horribly wrong for stores
51 * using r31 (i.e. XZR/WZR) as the source register.
53 #define QCOM_DUMMY_VAL -1
55 #define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
56 #define TLB_SPIN_COUNT 10
58 #define MSI_IOVA_BASE 0x8000000
59 #define MSI_IOVA_LENGTH 0x100000
61 static int force_stage;
63 * not really modular, but the easiest way to keep compat with existing
64 * bootargs behaviour is to continue using module_param() here.
66 module_param(force_stage, int, S_IRUGO);
67 MODULE_PARM_DESC(force_stage,
68 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
69 static bool disable_bypass =
70 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
71 module_param(disable_bypass, bool, S_IRUGO);
72 MODULE_PARM_DESC(disable_bypass,
73 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
75 struct arm_smmu_s2cr {
76 struct iommu_group *group;
78 enum arm_smmu_s2cr_type type;
79 enum arm_smmu_s2cr_privcfg privcfg;
83 #define s2cr_init_val (struct arm_smmu_s2cr){ \
84 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
97 struct arm_smmu_cfg *cfg;
100 struct arm_smmu_master_cfg {
101 struct arm_smmu_device *smmu;
104 #define INVALID_SMENDX -1
105 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
106 #define fwspec_smmu(fw) (__fwspec_cfg(fw)->smmu)
107 #define fwspec_smendx(fw, i) \
108 (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
109 #define for_each_cfg_sme(fw, i, idx) \
110 for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
112 static bool using_legacy_binding, using_generic_binding;
114 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
116 if (pm_runtime_enabled(smmu->dev))
117 return pm_runtime_resume_and_get(smmu->dev);
122 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
124 if (pm_runtime_enabled(smmu->dev))
125 pm_runtime_put(smmu->dev);
128 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
130 return container_of(dom, struct arm_smmu_domain, domain);
133 static struct device_node *dev_get_dev_node(struct device *dev)
135 if (dev_is_pci(dev)) {
136 struct pci_bus *bus = to_pci_dev(dev)->bus;
138 while (!pci_is_root_bus(bus))
140 return of_node_get(bus->bridge->parent->of_node);
143 return of_node_get(dev->of_node);
146 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
148 *((__be32 *)data) = cpu_to_be32(alias);
149 return 0; /* Continue walking */
152 static int __find_legacy_master_phandle(struct device *dev, void *data)
154 struct of_phandle_iterator *it = *(void **)data;
155 struct device_node *np = it->node;
158 of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
159 "#stream-id-cells", -1)
160 if (it->node == np) {
161 *(void **)data = dev;
165 return err == -ENOENT ? 0 : err;
168 static struct platform_driver arm_smmu_driver;
169 static struct iommu_ops arm_smmu_ops;
171 static int arm_smmu_register_legacy_master(struct device *dev,
172 struct arm_smmu_device **smmu)
174 struct device *smmu_dev;
175 struct device_node *np;
176 struct of_phandle_iterator it;
182 np = dev_get_dev_node(dev);
183 if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
189 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
190 __find_legacy_master_phandle);
198 if (dev_is_pci(dev)) {
199 /* "mmu-masters" assumes Stream ID == Requester ID */
200 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
206 err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
211 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
215 *smmu = dev_get_drvdata(smmu_dev);
216 of_phandle_iterator_args(&it, sids, it.cur_count);
217 err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
222 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
227 idx = find_next_zero_bit(map, end, start);
230 } while (test_and_set_bit(idx, map));
235 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
240 /* Wait for any pending TLB invalidations to complete */
241 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
242 int sync, int status)
244 unsigned int spin_cnt, delay;
247 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
248 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
249 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
250 reg = arm_smmu_readl(smmu, page, status);
251 if (!(reg & sTLBGSTATUS_GSACTIVE))
257 dev_err_ratelimited(smmu->dev,
258 "TLB sync timed out -- SMMU may be deadlocked\n");
261 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
265 spin_lock_irqsave(&smmu->global_sync_lock, flags);
266 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
267 ARM_SMMU_GR0_sTLBGSTATUS);
268 spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
271 static void arm_smmu_tlb_sync_context(void *cookie)
273 struct arm_smmu_domain *smmu_domain = cookie;
274 struct arm_smmu_device *smmu = smmu_domain->smmu;
277 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
278 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
279 ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
280 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
283 static void arm_smmu_tlb_sync_vmid(void *cookie)
285 struct arm_smmu_domain *smmu_domain = cookie;
287 arm_smmu_tlb_sync_global(smmu_domain->smmu);
290 static void arm_smmu_tlb_inv_context_s1(void *cookie)
292 struct arm_smmu_domain *smmu_domain = cookie;
294 * The TLBI write may be relaxed, so ensure that PTEs cleared by the
295 * current CPU are visible beforehand.
298 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
299 ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
300 arm_smmu_tlb_sync_context(cookie);
303 static void arm_smmu_tlb_inv_context_s2(void *cookie)
305 struct arm_smmu_domain *smmu_domain = cookie;
306 struct arm_smmu_device *smmu = smmu_domain->smmu;
310 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
311 arm_smmu_tlb_sync_global(smmu);
314 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
315 size_t granule, bool leaf, void *cookie)
317 struct arm_smmu_domain *smmu_domain = cookie;
318 struct arm_smmu_device *smmu = smmu_domain->smmu;
319 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
320 int reg, idx = cfg->cbndx;
322 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
325 reg = leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
327 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
328 iova = (iova >> 12) << 12;
331 arm_smmu_cb_write(smmu, idx, reg, iova);
333 } while (size -= granule);
336 iova |= (u64)cfg->asid << 48;
338 arm_smmu_cb_writeq(smmu, idx, reg, iova);
339 iova += granule >> 12;
340 } while (size -= granule);
344 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
345 size_t granule, bool leaf, void *cookie)
347 struct arm_smmu_domain *smmu_domain = cookie;
348 struct arm_smmu_device *smmu = smmu_domain->smmu;
349 int reg, idx = smmu_domain->cfg.cbndx;
351 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
354 reg = leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L : ARM_SMMU_CB_S2_TLBIIPAS2;
357 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
358 arm_smmu_cb_writeq(smmu, idx, reg, iova);
360 arm_smmu_cb_write(smmu, idx, reg, iova);
361 iova += granule >> 12;
362 } while (size -= granule);
366 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
367 * almost negligible, but the benefit of getting the first one in as far ahead
368 * of the sync as possible is significant, hence we don't just make this a
369 * no-op and set .tlb_sync to arm_smmu_tlb_inv_context_s2() as you might think.
371 static void arm_smmu_tlb_inv_vmid_nosync(unsigned long iova, size_t size,
372 size_t granule, bool leaf, void *cookie)
374 struct arm_smmu_domain *smmu_domain = cookie;
375 struct arm_smmu_device *smmu = smmu_domain->smmu;
377 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
380 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
383 static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
384 size_t granule, void *cookie)
386 struct arm_smmu_domain *smmu_domain = cookie;
387 const struct arm_smmu_flush_ops *ops = smmu_domain->flush_ops;
389 ops->tlb_inv_range(iova, size, granule, false, cookie);
390 ops->tlb_sync(cookie);
393 static void arm_smmu_tlb_inv_leaf(unsigned long iova, size_t size,
394 size_t granule, void *cookie)
396 struct arm_smmu_domain *smmu_domain = cookie;
397 const struct arm_smmu_flush_ops *ops = smmu_domain->flush_ops;
399 ops->tlb_inv_range(iova, size, granule, true, cookie);
400 ops->tlb_sync(cookie);
403 static void arm_smmu_tlb_add_page(struct iommu_iotlb_gather *gather,
404 unsigned long iova, size_t granule,
407 struct arm_smmu_domain *smmu_domain = cookie;
408 const struct arm_smmu_flush_ops *ops = smmu_domain->flush_ops;
410 ops->tlb_inv_range(iova, granule, granule, true, cookie);
413 static const struct arm_smmu_flush_ops arm_smmu_s1_tlb_ops = {
415 .tlb_flush_all = arm_smmu_tlb_inv_context_s1,
416 .tlb_flush_walk = arm_smmu_tlb_inv_walk,
417 .tlb_flush_leaf = arm_smmu_tlb_inv_leaf,
418 .tlb_add_page = arm_smmu_tlb_add_page,
420 .tlb_inv_range = arm_smmu_tlb_inv_range_s1,
421 .tlb_sync = arm_smmu_tlb_sync_context,
424 static const struct arm_smmu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
426 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
427 .tlb_flush_walk = arm_smmu_tlb_inv_walk,
428 .tlb_flush_leaf = arm_smmu_tlb_inv_leaf,
429 .tlb_add_page = arm_smmu_tlb_add_page,
431 .tlb_inv_range = arm_smmu_tlb_inv_range_s2,
432 .tlb_sync = arm_smmu_tlb_sync_context,
435 static const struct arm_smmu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
437 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
438 .tlb_flush_walk = arm_smmu_tlb_inv_walk,
439 .tlb_flush_leaf = arm_smmu_tlb_inv_leaf,
440 .tlb_add_page = arm_smmu_tlb_add_page,
442 .tlb_inv_range = arm_smmu_tlb_inv_vmid_nosync,
443 .tlb_sync = arm_smmu_tlb_sync_vmid,
446 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
448 u32 fsr, fsynr, cbfrsynra;
450 struct iommu_domain *domain = dev;
451 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
452 struct arm_smmu_device *smmu = smmu_domain->smmu;
453 int idx = smmu_domain->cfg.cbndx;
455 fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
456 if (!(fsr & FSR_FAULT))
459 fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
460 iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
461 cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
463 dev_err_ratelimited(smmu->dev,
464 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
465 fsr, iova, fsynr, cbfrsynra, idx);
467 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
471 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
473 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
474 struct arm_smmu_device *smmu = dev;
476 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
477 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
478 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
479 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
484 dev_err_ratelimited(smmu->dev,
485 "Unexpected global fault, this could be serious\n");
486 dev_err_ratelimited(smmu->dev,
487 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
488 gfsr, gfsynr0, gfsynr1, gfsynr2);
490 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
494 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
495 struct io_pgtable_cfg *pgtbl_cfg)
497 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
498 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
499 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
505 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
506 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
508 cb->tcr[0] = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
509 cb->tcr[1] = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
510 cb->tcr[1] |= FIELD_PREP(TCR2_SEP, TCR2_SEP_UPSTREAM);
511 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
512 cb->tcr[1] |= TCR2_AS;
515 cb->tcr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
520 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
521 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
522 cb->ttbr[1] = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
524 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
525 cb->ttbr[0] |= FIELD_PREP(TTBRn_ASID, cfg->asid);
526 cb->ttbr[1] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
527 cb->ttbr[1] |= FIELD_PREP(TTBRn_ASID, cfg->asid);
530 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
533 /* MAIRs (stage-1 only) */
535 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
536 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
537 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
539 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
540 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
545 static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
549 struct arm_smmu_cb *cb = &smmu->cbs[idx];
550 struct arm_smmu_cfg *cfg = cb->cfg;
552 /* Unassigned context banks only need disabling */
554 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
558 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
561 if (smmu->version > ARM_SMMU_V1) {
562 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
566 /* 16-bit VMIDs live in CBA2R */
567 if (smmu->features & ARM_SMMU_FEAT_VMID16)
568 reg |= FIELD_PREP(CBA2R_VMID16, cfg->vmid);
570 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
574 reg = FIELD_PREP(CBAR_TYPE, cfg->cbar);
575 if (smmu->version < ARM_SMMU_V2)
576 reg |= FIELD_PREP(CBAR_IRPTNDX, cfg->irptndx);
579 * Use the weakest shareability/memory types, so they are
580 * overridden by the ttbcr/pte.
583 reg |= FIELD_PREP(CBAR_S1_BPSHCFG, CBAR_S1_BPSHCFG_NSH) |
584 FIELD_PREP(CBAR_S1_MEMATTR, CBAR_S1_MEMATTR_WB);
585 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
586 /* 8-bit VMIDs live in CBAR */
587 reg |= FIELD_PREP(CBAR_VMID, cfg->vmid);
589 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
593 * We must write this before the TTBRs, since it determines the
594 * access behaviour of some fields (in particular, ASID[15:8]).
596 if (stage1 && smmu->version > ARM_SMMU_V1)
597 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
598 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
601 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
602 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
603 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
604 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
606 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
608 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
612 /* MAIRs (stage-1 only) */
614 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
615 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
619 reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
621 reg |= SCTLR_S1_ASIDPNE;
622 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
625 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
628 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
629 struct arm_smmu_device *smmu)
631 int irq, start, ret = 0;
632 unsigned long ias, oas;
633 struct io_pgtable_ops *pgtbl_ops;
634 struct io_pgtable_cfg pgtbl_cfg;
635 enum io_pgtable_fmt fmt;
636 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
637 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
639 mutex_lock(&smmu_domain->init_mutex);
640 if (smmu_domain->smmu)
643 if (domain->type == IOMMU_DOMAIN_IDENTITY) {
644 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
645 smmu_domain->smmu = smmu;
650 * Mapping the requested stage onto what we support is surprisingly
651 * complicated, mainly because the spec allows S1+S2 SMMUs without
652 * support for nested translation. That means we end up with the
655 * Requested Supported Actual
665 * Note that you can't actually request stage-2 mappings.
667 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
668 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
669 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
670 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
673 * Choosing a suitable context format is even more fiddly. Until we
674 * grow some way for the caller to express a preference, and/or move
675 * the decision into the io-pgtable code where it arguably belongs,
676 * just aim for the closest thing to the rest of the system, and hope
677 * that the hardware isn't esoteric enough that we can't assume AArch64
678 * support to be a superset of AArch32 support...
680 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
681 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
682 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
683 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
684 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
685 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
686 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
687 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
688 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
689 ARM_SMMU_FEAT_FMT_AARCH64_16K |
690 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
691 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
693 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
698 switch (smmu_domain->stage) {
699 case ARM_SMMU_DOMAIN_S1:
700 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
701 start = smmu->num_s2_context_banks;
703 oas = smmu->ipa_size;
704 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
705 fmt = ARM_64_LPAE_S1;
706 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
707 fmt = ARM_32_LPAE_S1;
708 ias = min(ias, 32UL);
709 oas = min(oas, 40UL);
712 ias = min(ias, 32UL);
713 oas = min(oas, 32UL);
715 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
717 case ARM_SMMU_DOMAIN_NESTED:
719 * We will likely want to change this if/when KVM gets
722 case ARM_SMMU_DOMAIN_S2:
723 cfg->cbar = CBAR_TYPE_S2_TRANS;
725 ias = smmu->ipa_size;
727 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
728 fmt = ARM_64_LPAE_S2;
730 fmt = ARM_32_LPAE_S2;
731 ias = min(ias, 40UL);
732 oas = min(oas, 40UL);
734 if (smmu->version == ARM_SMMU_V2)
735 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
737 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
743 ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
744 smmu->num_context_banks);
749 if (smmu->version < ARM_SMMU_V2) {
750 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
751 cfg->irptndx %= smmu->num_context_irqs;
753 cfg->irptndx = cfg->cbndx;
756 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
757 cfg->vmid = cfg->cbndx + 1;
759 cfg->asid = cfg->cbndx;
761 smmu_domain->smmu = smmu;
762 if (smmu->impl && smmu->impl->init_context) {
763 ret = smmu->impl->init_context(smmu_domain);
768 pgtbl_cfg = (struct io_pgtable_cfg) {
769 .pgsize_bitmap = smmu->pgsize_bitmap,
772 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
773 .tlb = &smmu_domain->flush_ops->tlb,
774 .iommu_dev = smmu->dev,
777 if (smmu_domain->non_strict)
778 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
780 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
786 /* Update the domain's page sizes to reflect the page table format */
787 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
788 domain->geometry.aperture_end = (1UL << ias) - 1;
789 domain->geometry.force_aperture = true;
791 /* Initialise the context bank with our page table cfg */
792 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
793 arm_smmu_write_context_bank(smmu, cfg->cbndx);
796 * Request context fault interrupt. Do this last to avoid the
797 * handler seeing a half-initialised domain state.
799 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
800 ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
801 IRQF_SHARED, "arm-smmu-context-fault", domain);
803 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
805 cfg->irptndx = INVALID_IRPTNDX;
808 mutex_unlock(&smmu_domain->init_mutex);
810 /* Publish page table ops for map/unmap */
811 smmu_domain->pgtbl_ops = pgtbl_ops;
815 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
816 smmu_domain->smmu = NULL;
818 mutex_unlock(&smmu_domain->init_mutex);
822 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
824 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
825 struct arm_smmu_device *smmu = smmu_domain->smmu;
826 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
829 if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
832 ret = arm_smmu_rpm_get(smmu);
837 * Disable the context bank and free the page tables before freeing
840 smmu->cbs[cfg->cbndx].cfg = NULL;
841 arm_smmu_write_context_bank(smmu, cfg->cbndx);
843 if (cfg->irptndx != INVALID_IRPTNDX) {
844 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
845 devm_free_irq(smmu->dev, irq, domain);
848 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
849 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
851 arm_smmu_rpm_put(smmu);
854 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
856 struct arm_smmu_domain *smmu_domain;
858 if (type != IOMMU_DOMAIN_UNMANAGED &&
859 type != IOMMU_DOMAIN_DMA &&
860 type != IOMMU_DOMAIN_IDENTITY)
863 * Allocate the domain and initialise some of its data structures.
864 * We can't really do anything meaningful until we've added a
867 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
871 if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
872 iommu_get_dma_cookie(&smmu_domain->domain))) {
877 mutex_init(&smmu_domain->init_mutex);
878 spin_lock_init(&smmu_domain->cb_lock);
880 return &smmu_domain->domain;
883 static void arm_smmu_domain_free(struct iommu_domain *domain)
885 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
888 * Free the domain resources. We assume that all devices have
889 * already been detached.
891 iommu_put_dma_cookie(domain);
892 arm_smmu_destroy_domain_context(domain);
896 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
898 struct arm_smmu_smr *smr = smmu->smrs + idx;
899 u32 reg = FIELD_PREP(SMR_ID, smr->id) | FIELD_PREP(SMR_MASK, smr->mask);
901 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
903 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
906 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
908 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
909 u32 reg = FIELD_PREP(S2CR_TYPE, s2cr->type) |
910 FIELD_PREP(S2CR_CBNDX, s2cr->cbndx) |
911 FIELD_PREP(S2CR_PRIVCFG, s2cr->privcfg);
913 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
914 smmu->smrs[idx].valid)
915 reg |= S2CR_EXIDVALID;
916 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
919 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
921 arm_smmu_write_s2cr(smmu, idx);
923 arm_smmu_write_smr(smmu, idx);
927 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
928 * should be called after sCR0 is written.
930 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
938 * SMR.ID bits may not be preserved if the corresponding MASK
939 * bits are set, so check each one separately. We can reject
940 * masters later if they try to claim IDs outside these masks.
942 smr = FIELD_PREP(SMR_ID, smmu->streamid_mask);
943 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(0), smr);
944 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(0));
945 smmu->streamid_mask = FIELD_GET(SMR_ID, smr);
947 smr = FIELD_PREP(SMR_MASK, smmu->streamid_mask);
948 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(0), smr);
949 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(0));
950 smmu->smr_mask_mask = FIELD_GET(SMR_MASK, smr);
953 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
955 struct arm_smmu_smr *smrs = smmu->smrs;
956 int i, free_idx = -ENOSPC;
958 /* Stream indexing is blissfully easy */
962 /* Validating SMRs is... less so */
963 for (i = 0; i < smmu->num_mapping_groups; ++i) {
964 if (!smrs[i].valid) {
966 * Note the first free entry we come across, which
967 * we'll claim in the end if nothing else matches.
974 * If the new entry is _entirely_ matched by an existing entry,
975 * then reuse that, with the guarantee that there also cannot
976 * be any subsequent conflicting entries. In normal use we'd
977 * expect simply identical entries for this case, but there's
978 * no harm in accommodating the generalisation.
980 if ((mask & smrs[i].mask) == mask &&
981 !((id ^ smrs[i].id) & ~smrs[i].mask))
984 * If the new entry has any other overlap with an existing one,
985 * though, then there always exists at least one stream ID
986 * which would cause a conflict, and we can't allow that risk.
988 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
995 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
997 if (--smmu->s2crs[idx].count)
1000 smmu->s2crs[idx] = s2cr_init_val;
1002 smmu->smrs[idx].valid = false;
1007 static int arm_smmu_master_alloc_smes(struct device *dev)
1009 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1010 struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1011 struct arm_smmu_device *smmu = cfg->smmu;
1012 struct arm_smmu_smr *smrs = smmu->smrs;
1013 struct iommu_group *group;
1016 mutex_lock(&smmu->stream_map_mutex);
1017 /* Figure out a viable stream map entry allocation */
1018 for_each_cfg_sme(fwspec, i, idx) {
1019 u16 sid = FIELD_GET(SMR_ID, fwspec->ids[i]);
1020 u16 mask = FIELD_GET(SMR_MASK, fwspec->ids[i]);
1022 if (idx != INVALID_SMENDX) {
1027 ret = arm_smmu_find_sme(smmu, sid, mask);
1032 if (smrs && smmu->s2crs[idx].count == 0) {
1034 smrs[idx].mask = mask;
1035 smrs[idx].valid = true;
1037 smmu->s2crs[idx].count++;
1038 cfg->smendx[i] = (s16)idx;
1041 group = iommu_group_get_for_dev(dev);
1043 group = ERR_PTR(-ENOMEM);
1044 if (IS_ERR(group)) {
1045 ret = PTR_ERR(group);
1048 iommu_group_put(group);
1050 /* It worked! Now, poke the actual hardware */
1051 for_each_cfg_sme(fwspec, i, idx) {
1052 arm_smmu_write_sme(smmu, idx);
1053 smmu->s2crs[idx].group = group;
1056 mutex_unlock(&smmu->stream_map_mutex);
1061 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1062 cfg->smendx[i] = INVALID_SMENDX;
1064 mutex_unlock(&smmu->stream_map_mutex);
1068 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1070 struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1071 struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1074 mutex_lock(&smmu->stream_map_mutex);
1075 for_each_cfg_sme(fwspec, i, idx) {
1076 if (arm_smmu_free_sme(smmu, idx))
1077 arm_smmu_write_sme(smmu, idx);
1078 cfg->smendx[i] = INVALID_SMENDX;
1080 mutex_unlock(&smmu->stream_map_mutex);
1083 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1084 struct iommu_fwspec *fwspec)
1086 struct arm_smmu_device *smmu = smmu_domain->smmu;
1087 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1088 u8 cbndx = smmu_domain->cfg.cbndx;
1089 enum arm_smmu_s2cr_type type;
1092 if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1093 type = S2CR_TYPE_BYPASS;
1095 type = S2CR_TYPE_TRANS;
1097 for_each_cfg_sme(fwspec, i, idx) {
1098 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1101 s2cr[idx].type = type;
1102 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1103 s2cr[idx].cbndx = cbndx;
1104 arm_smmu_write_s2cr(smmu, idx);
1109 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1112 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1113 struct arm_smmu_device *smmu;
1114 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1116 if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1117 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1122 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1123 * domains between of_xlate() and add_device() - we have no way to cope
1124 * with that, so until ARM gets converted to rely on groups and default
1125 * domains, just say no (but more politely than by dereferencing NULL).
1126 * This should be at least a WARN_ON once that's sorted.
1128 if (!fwspec->iommu_priv)
1131 smmu = fwspec_smmu(fwspec);
1133 ret = arm_smmu_rpm_get(smmu);
1137 /* Ensure that the domain is finalised */
1138 ret = arm_smmu_init_domain_context(domain, smmu);
1143 * Sanity check the domain. We don't support domains across
1146 if (smmu_domain->smmu != smmu) {
1148 "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1149 dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1154 /* Looks ok, so add the device to the domain */
1155 ret = arm_smmu_domain_add_master(smmu_domain, fwspec);
1158 arm_smmu_rpm_put(smmu);
1162 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1163 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
1165 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1166 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1172 arm_smmu_rpm_get(smmu);
1173 ret = ops->map(ops, iova, paddr, size, prot);
1174 arm_smmu_rpm_put(smmu);
1179 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1180 size_t size, struct iommu_iotlb_gather *gather)
1182 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1183 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1189 arm_smmu_rpm_get(smmu);
1190 ret = ops->unmap(ops, iova, size, gather);
1191 arm_smmu_rpm_put(smmu);
1196 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1198 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1199 struct arm_smmu_device *smmu = smmu_domain->smmu;
1201 if (smmu_domain->flush_ops) {
1202 arm_smmu_rpm_get(smmu);
1203 smmu_domain->flush_ops->tlb.tlb_flush_all(smmu_domain);
1204 arm_smmu_rpm_put(smmu);
1208 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1209 struct iommu_iotlb_gather *gather)
1211 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1212 struct arm_smmu_device *smmu = smmu_domain->smmu;
1214 if (smmu_domain->flush_ops) {
1215 arm_smmu_rpm_get(smmu);
1216 smmu_domain->flush_ops->tlb_sync(smmu_domain);
1217 arm_smmu_rpm_put(smmu);
1221 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1224 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1225 struct arm_smmu_device *smmu = smmu_domain->smmu;
1226 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1227 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1228 struct device *dev = smmu->dev;
1232 unsigned long va, flags;
1233 int ret, idx = cfg->cbndx;
1234 phys_addr_t addr = 0;
1236 ret = arm_smmu_rpm_get(smmu);
1240 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1241 va = iova & ~0xfffUL;
1242 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1243 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1245 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1247 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1248 if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ATSR_ACTIVE), 5, 50)) {
1249 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1251 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1253 arm_smmu_rpm_put(smmu);
1254 return ops->iova_to_phys(ops, iova);
1257 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1258 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1259 if (phys & CB_PAR_F) {
1260 dev_err(dev, "translation fault!\n");
1261 dev_err(dev, "PAR = 0x%llx\n", phys);
1265 addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1267 arm_smmu_rpm_put(smmu);
1272 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1275 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1276 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1278 if (domain->type == IOMMU_DOMAIN_IDENTITY)
1284 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1285 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1286 return arm_smmu_iova_to_phys_hard(domain, iova);
1288 return ops->iova_to_phys(ops, iova);
1291 static bool arm_smmu_capable(enum iommu_cap cap)
1294 case IOMMU_CAP_CACHE_COHERENCY:
1296 * Return true here as the SMMU can always send out coherent
1300 case IOMMU_CAP_NOEXEC:
1308 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1310 struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
1313 return dev ? dev_get_drvdata(dev) : NULL;
1316 static int arm_smmu_add_device(struct device *dev)
1318 struct arm_smmu_device *smmu;
1319 struct arm_smmu_master_cfg *cfg;
1320 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1323 if (using_legacy_binding) {
1324 ret = arm_smmu_register_legacy_master(dev, &smmu);
1327 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1328 * will allocate/initialise a new one. Thus we need to update fwspec for
1331 fwspec = dev_iommu_fwspec_get(dev);
1334 } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1335 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1341 for (i = 0; i < fwspec->num_ids; i++) {
1342 u16 sid = FIELD_GET(SMR_ID, fwspec->ids[i]);
1343 u16 mask = FIELD_GET(SMR_MASK, fwspec->ids[i]);
1345 if (sid & ~smmu->streamid_mask) {
1346 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1347 sid, smmu->streamid_mask);
1350 if (mask & ~smmu->smr_mask_mask) {
1351 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1352 mask, smmu->smr_mask_mask);
1358 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1364 fwspec->iommu_priv = cfg;
1366 cfg->smendx[i] = INVALID_SMENDX;
1368 ret = arm_smmu_rpm_get(smmu);
1372 ret = arm_smmu_master_alloc_smes(dev);
1373 arm_smmu_rpm_put(smmu);
1378 iommu_device_link(&smmu->iommu, dev);
1380 device_link_add(dev, smmu->dev,
1381 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1388 iommu_fwspec_free(dev);
1392 static void arm_smmu_remove_device(struct device *dev)
1394 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1395 struct arm_smmu_master_cfg *cfg;
1396 struct arm_smmu_device *smmu;
1399 if (!fwspec || fwspec->ops != &arm_smmu_ops)
1402 cfg = fwspec->iommu_priv;
1405 ret = arm_smmu_rpm_get(smmu);
1409 iommu_device_unlink(&smmu->iommu, dev);
1410 arm_smmu_master_free_smes(fwspec);
1412 arm_smmu_rpm_put(smmu);
1414 iommu_group_remove_device(dev);
1415 kfree(fwspec->iommu_priv);
1416 iommu_fwspec_free(dev);
1419 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1421 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1422 struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1423 struct iommu_group *group = NULL;
1426 for_each_cfg_sme(fwspec, i, idx) {
1427 if (group && smmu->s2crs[idx].group &&
1428 group != smmu->s2crs[idx].group)
1429 return ERR_PTR(-EINVAL);
1431 group = smmu->s2crs[idx].group;
1435 return iommu_group_ref_get(group);
1437 if (dev_is_pci(dev))
1438 group = pci_device_group(dev);
1439 else if (dev_is_fsl_mc(dev))
1440 group = fsl_mc_device_group(dev);
1442 group = generic_device_group(dev);
1447 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1448 enum iommu_attr attr, void *data)
1450 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1452 switch(domain->type) {
1453 case IOMMU_DOMAIN_UNMANAGED:
1455 case DOMAIN_ATTR_NESTING:
1456 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1462 case IOMMU_DOMAIN_DMA:
1464 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1465 *(int *)data = smmu_domain->non_strict;
1476 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1477 enum iommu_attr attr, void *data)
1480 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1482 mutex_lock(&smmu_domain->init_mutex);
1484 switch(domain->type) {
1485 case IOMMU_DOMAIN_UNMANAGED:
1487 case DOMAIN_ATTR_NESTING:
1488 if (smmu_domain->smmu) {
1494 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1496 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1502 case IOMMU_DOMAIN_DMA:
1504 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1505 smmu_domain->non_strict = *(int *)data;
1515 mutex_unlock(&smmu_domain->init_mutex);
1519 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1523 if (args->args_count > 0)
1524 fwid |= FIELD_PREP(SMR_ID, args->args[0]);
1526 if (args->args_count > 1)
1527 fwid |= FIELD_PREP(SMR_MASK, args->args[1]);
1528 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1529 fwid |= FIELD_PREP(SMR_MASK, mask);
1531 return iommu_fwspec_add_ids(dev, &fwid, 1);
1534 static void arm_smmu_get_resv_regions(struct device *dev,
1535 struct list_head *head)
1537 struct iommu_resv_region *region;
1538 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1540 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1541 prot, IOMMU_RESV_SW_MSI);
1545 list_add_tail(®ion->list, head);
1547 iommu_dma_get_resv_regions(dev, head);
1550 static void arm_smmu_put_resv_regions(struct device *dev,
1551 struct list_head *head)
1553 struct iommu_resv_region *entry, *next;
1555 list_for_each_entry_safe(entry, next, head, list)
1559 static struct iommu_ops arm_smmu_ops = {
1560 .capable = arm_smmu_capable,
1561 .domain_alloc = arm_smmu_domain_alloc,
1562 .domain_free = arm_smmu_domain_free,
1563 .attach_dev = arm_smmu_attach_dev,
1564 .map = arm_smmu_map,
1565 .unmap = arm_smmu_unmap,
1566 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
1567 .iotlb_sync = arm_smmu_iotlb_sync,
1568 .iova_to_phys = arm_smmu_iova_to_phys,
1569 .add_device = arm_smmu_add_device,
1570 .remove_device = arm_smmu_remove_device,
1571 .device_group = arm_smmu_device_group,
1572 .domain_get_attr = arm_smmu_domain_get_attr,
1573 .domain_set_attr = arm_smmu_domain_set_attr,
1574 .of_xlate = arm_smmu_of_xlate,
1575 .get_resv_regions = arm_smmu_get_resv_regions,
1576 .put_resv_regions = arm_smmu_put_resv_regions,
1577 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1580 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1585 /* clear global FSR */
1586 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1587 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1590 * Reset stream mapping groups: Initial values mark all SMRn as
1591 * invalid and all S2CRn as bypass unless overridden.
1593 for (i = 0; i < smmu->num_mapping_groups; ++i)
1594 arm_smmu_write_sme(smmu, i);
1596 /* Make sure all context banks are disabled and clear CB_FSR */
1597 for (i = 0; i < smmu->num_context_banks; ++i) {
1598 arm_smmu_write_context_bank(smmu, i);
1599 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, FSR_FAULT);
1602 /* Invalidate the TLB, just in case */
1603 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1604 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1606 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1608 /* Enable fault reporting */
1609 reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1611 /* Disable TLB broadcasting. */
1612 reg |= (sCR0_VMIDPNE | sCR0_PTM);
1614 /* Enable client access, handling unmatched streams as appropriate */
1615 reg &= ~sCR0_CLIENTPD;
1619 reg &= ~sCR0_USFCFG;
1621 /* Disable forced broadcasting */
1624 /* Don't upgrade barriers */
1627 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1628 reg |= sCR0_VMID16EN;
1630 if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1631 reg |= sCR0_EXIDENABLE;
1633 if (smmu->impl && smmu->impl->reset)
1634 smmu->impl->reset(smmu);
1636 /* Push the button */
1637 arm_smmu_tlb_sync_global(smmu);
1638 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1641 static int arm_smmu_id_size_to_bits(int size)
1660 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1664 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1667 dev_notice(smmu->dev, "probing hardware configuration...\n");
1668 dev_notice(smmu->dev, "SMMUv%d with:\n",
1669 smmu->version == ARM_SMMU_V2 ? 2 : 1);
1672 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1674 /* Restrict available stages based on module parameter */
1675 if (force_stage == 1)
1676 id &= ~(ID0_S2TS | ID0_NTS);
1677 else if (force_stage == 2)
1678 id &= ~(ID0_S1TS | ID0_NTS);
1680 if (id & ID0_S1TS) {
1681 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1682 dev_notice(smmu->dev, "\tstage 1 translation\n");
1685 if (id & ID0_S2TS) {
1686 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1687 dev_notice(smmu->dev, "\tstage 2 translation\n");
1691 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1692 dev_notice(smmu->dev, "\tnested translation\n");
1695 if (!(smmu->features &
1696 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1697 dev_err(smmu->dev, "\tno translation support!\n");
1701 if ((id & ID0_S1TS) &&
1702 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1703 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1704 dev_notice(smmu->dev, "\taddress translation ops\n");
1708 * In order for DMA API calls to work properly, we must defer to what
1709 * the FW says about coherency, regardless of what the hardware claims.
1710 * Fortunately, this also opens up a workaround for systems where the
1711 * ID register value has ended up configured incorrectly.
1713 cttw_reg = !!(id & ID0_CTTW);
1714 if (cttw_fw || cttw_reg)
1715 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1716 cttw_fw ? "" : "non-");
1717 if (cttw_fw != cttw_reg)
1718 dev_notice(smmu->dev,
1719 "\t(IDR0.CTTW overridden by FW configuration)\n");
1721 /* Max. number of entries we have for stream matching/indexing */
1722 if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) {
1723 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1726 size = 1 << FIELD_GET(ID0_NUMSIDB, id);
1728 smmu->streamid_mask = size - 1;
1730 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1731 size = FIELD_GET(ID0_NUMSMRG, id);
1734 "stream-matching supported, but no SMRs present!\n");
1738 /* Zero-initialised to mark as invalid */
1739 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1744 dev_notice(smmu->dev,
1745 "\tstream matching with %u register groups", size);
1747 /* s2cr->type == 0 means translation, so initialise explicitly */
1748 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1752 for (i = 0; i < size; i++)
1753 smmu->s2crs[i] = s2cr_init_val;
1755 smmu->num_mapping_groups = size;
1756 mutex_init(&smmu->stream_map_mutex);
1757 spin_lock_init(&smmu->global_sync_lock);
1759 if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1760 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1761 if (!(id & ID0_PTFS_NO_AARCH32S))
1762 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1766 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1767 smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1769 /* Check for size mismatch of SMMU address space from mapped region */
1770 size = 1 << (FIELD_GET(ID1_NUMPAGENDXB, id) + 1);
1771 if (smmu->numpage != 2 * size << smmu->pgshift)
1773 "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1774 2 * size << smmu->pgshift, smmu->numpage);
1775 /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1776 smmu->numpage = size;
1778 smmu->num_s2_context_banks = FIELD_GET(ID1_NUMS2CB, id);
1779 smmu->num_context_banks = FIELD_GET(ID1_NUMCB, id);
1780 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1781 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1784 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1785 smmu->num_context_banks, smmu->num_s2_context_banks);
1786 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1787 sizeof(*smmu->cbs), GFP_KERNEL);
1792 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1793 size = arm_smmu_id_size_to_bits(FIELD_GET(ID2_IAS, id));
1794 smmu->ipa_size = size;
1796 /* The output mask is also applied for bypass */
1797 size = arm_smmu_id_size_to_bits(FIELD_GET(ID2_OAS, id));
1798 smmu->pa_size = size;
1800 if (id & ID2_VMID16)
1801 smmu->features |= ARM_SMMU_FEAT_VMID16;
1804 * What the page table walker can address actually depends on which
1805 * descriptor format is in use, but since a) we don't know that yet,
1806 * and b) it can vary per context bank, this will have to do...
1808 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1810 "failed to set DMA mask for table walker\n");
1812 if (smmu->version < ARM_SMMU_V2) {
1813 smmu->va_size = smmu->ipa_size;
1814 if (smmu->version == ARM_SMMU_V1_64K)
1815 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1817 size = FIELD_GET(ID2_UBS, id);
1818 smmu->va_size = arm_smmu_id_size_to_bits(size);
1819 if (id & ID2_PTFS_4K)
1820 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1821 if (id & ID2_PTFS_16K)
1822 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1823 if (id & ID2_PTFS_64K)
1824 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1827 /* Now we've corralled the various formats, what'll it do? */
1828 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1829 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1830 if (smmu->features &
1831 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1832 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1833 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1834 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1835 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1836 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1838 if (arm_smmu_ops.pgsize_bitmap == -1UL)
1839 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1841 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1842 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1843 smmu->pgsize_bitmap);
1846 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1847 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1848 smmu->va_size, smmu->ipa_size);
1850 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1851 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1852 smmu->ipa_size, smmu->pa_size);
1854 if (smmu->impl && smmu->impl->cfg_probe)
1855 return smmu->impl->cfg_probe(smmu);
1860 struct arm_smmu_match_data {
1861 enum arm_smmu_arch_version version;
1862 enum arm_smmu_implementation model;
1865 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1866 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1868 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1869 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1870 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1871 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1872 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1873 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1875 static const struct of_device_id arm_smmu_of_match[] = {
1876 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1877 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1878 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1879 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1880 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1881 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1882 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1887 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1892 case ACPI_IORT_SMMU_V1:
1893 case ACPI_IORT_SMMU_CORELINK_MMU400:
1894 smmu->version = ARM_SMMU_V1;
1895 smmu->model = GENERIC_SMMU;
1897 case ACPI_IORT_SMMU_CORELINK_MMU401:
1898 smmu->version = ARM_SMMU_V1_64K;
1899 smmu->model = GENERIC_SMMU;
1901 case ACPI_IORT_SMMU_V2:
1902 smmu->version = ARM_SMMU_V2;
1903 smmu->model = GENERIC_SMMU;
1905 case ACPI_IORT_SMMU_CORELINK_MMU500:
1906 smmu->version = ARM_SMMU_V2;
1907 smmu->model = ARM_MMU500;
1909 case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1910 smmu->version = ARM_SMMU_V2;
1911 smmu->model = CAVIUM_SMMUV2;
1920 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1921 struct arm_smmu_device *smmu)
1923 struct device *dev = smmu->dev;
1924 struct acpi_iort_node *node =
1925 *(struct acpi_iort_node **)dev_get_platdata(dev);
1926 struct acpi_iort_smmu *iort_smmu;
1929 /* Retrieve SMMU1/2 specific data */
1930 iort_smmu = (struct acpi_iort_smmu *)node->node_data;
1932 ret = acpi_smmu_get_data(iort_smmu->model, smmu);
1936 /* Ignore the configuration access interrupt */
1937 smmu->num_global_irqs = 1;
1939 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
1940 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1945 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1946 struct arm_smmu_device *smmu)
1952 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
1953 struct arm_smmu_device *smmu)
1955 const struct arm_smmu_match_data *data;
1956 struct device *dev = &pdev->dev;
1957 bool legacy_binding;
1959 if (of_property_read_u32(dev->of_node, "#global-interrupts",
1960 &smmu->num_global_irqs)) {
1961 dev_err(dev, "missing #global-interrupts property\n");
1965 data = of_device_get_match_data(dev);
1966 smmu->version = data->version;
1967 smmu->model = data->model;
1969 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
1970 if (legacy_binding && !using_generic_binding) {
1971 if (!using_legacy_binding)
1972 pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
1973 using_legacy_binding = true;
1974 } else if (!legacy_binding && !using_legacy_binding) {
1975 using_generic_binding = true;
1977 dev_err(dev, "not probing due to mismatched DT properties\n");
1981 if (of_dma_is_coherent(dev->of_node))
1982 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1987 static void arm_smmu_bus_init(void)
1989 /* Oh, for a proper bus abstraction */
1990 if (!iommu_present(&platform_bus_type))
1991 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
1992 #ifdef CONFIG_ARM_AMBA
1993 if (!iommu_present(&amba_bustype))
1994 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
1997 if (!iommu_present(&pci_bus_type)) {
1999 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2002 #ifdef CONFIG_FSL_MC_BUS
2003 if (!iommu_present(&fsl_mc_bus_type))
2004 bus_set_iommu(&fsl_mc_bus_type, &arm_smmu_ops);
2008 static int arm_smmu_device_probe(struct platform_device *pdev)
2010 struct resource *res;
2011 resource_size_t ioaddr;
2012 struct arm_smmu_device *smmu;
2013 struct device *dev = &pdev->dev;
2014 int num_irqs, i, err;
2016 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2018 dev_err(dev, "failed to allocate arm_smmu_device\n");
2024 err = arm_smmu_device_dt_probe(pdev, smmu);
2026 err = arm_smmu_device_acpi_probe(pdev, smmu);
2031 smmu = arm_smmu_impl_init(smmu);
2033 return PTR_ERR(smmu);
2035 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2036 ioaddr = res->start;
2037 smmu->base = devm_ioremap_resource(dev, res);
2038 if (IS_ERR(smmu->base))
2039 return PTR_ERR(smmu->base);
2041 * The resource size should effectively match the value of SMMU_TOP;
2042 * stash that temporarily until we know PAGESIZE to validate it with.
2044 smmu->numpage = resource_size(res);
2047 while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2049 if (num_irqs > smmu->num_global_irqs)
2050 smmu->num_context_irqs++;
2053 if (!smmu->num_context_irqs) {
2054 dev_err(dev, "found %d interrupts but expected at least %d\n",
2055 num_irqs, smmu->num_global_irqs + 1);
2059 smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),
2062 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2066 for (i = 0; i < num_irqs; ++i) {
2067 int irq = platform_get_irq(pdev, i);
2070 dev_err(dev, "failed to get irq index %d\n", i);
2073 smmu->irqs[i] = irq;
2076 err = devm_clk_bulk_get_all(dev, &smmu->clks);
2078 dev_err(dev, "failed to get clocks %d\n", err);
2081 smmu->num_clks = err;
2083 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2087 err = arm_smmu_device_cfg_probe(smmu);
2091 if (smmu->version == ARM_SMMU_V2) {
2092 if (smmu->num_context_banks > smmu->num_context_irqs) {
2094 "found only %d context irq(s) but %d required\n",
2095 smmu->num_context_irqs, smmu->num_context_banks);
2099 /* Ignore superfluous interrupts */
2100 smmu->num_context_irqs = smmu->num_context_banks;
2103 for (i = 0; i < smmu->num_global_irqs; ++i) {
2104 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2105 arm_smmu_global_fault,
2107 "arm-smmu global fault",
2110 dev_err(dev, "failed to request global IRQ %d (%u)\n",
2116 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2117 "smmu.%pa", &ioaddr);
2119 dev_err(dev, "Failed to register iommu in sysfs\n");
2123 iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2124 iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2126 err = iommu_device_register(&smmu->iommu);
2128 dev_err(dev, "Failed to register iommu\n");
2132 platform_set_drvdata(pdev, smmu);
2133 arm_smmu_device_reset(smmu);
2134 arm_smmu_test_smr_masks(smmu);
2137 * We want to avoid touching dev->power.lock in fastpaths unless
2138 * it's really going to do something useful - pm_runtime_enabled()
2139 * can serve as an ideal proxy for that decision. So, conditionally
2140 * enable pm_runtime.
2142 if (dev->pm_domain) {
2143 pm_runtime_set_active(dev);
2144 pm_runtime_enable(dev);
2148 * For ACPI and generic DT bindings, an SMMU will be probed before
2149 * any device which might need it, so we want the bus ops in place
2150 * ready to handle default domain setup as soon as any SMMU exists.
2152 if (!using_legacy_binding)
2153 arm_smmu_bus_init();
2159 * With the legacy DT binding in play, though, we have no guarantees about
2160 * probe order, but then we're also not doing default domains, so we can
2161 * delay setting bus ops until we're sure every possible SMMU is ready,
2162 * and that way ensure that no add_device() calls get missed.
2164 static int arm_smmu_legacy_bus_init(void)
2166 if (using_legacy_binding)
2167 arm_smmu_bus_init();
2170 device_initcall_sync(arm_smmu_legacy_bus_init);
2172 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2174 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2179 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2180 dev_err(&pdev->dev, "removing device with active domains!\n");
2182 arm_smmu_rpm_get(smmu);
2183 /* Turn the thing off */
2184 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, sCR0_CLIENTPD);
2185 arm_smmu_rpm_put(smmu);
2187 if (pm_runtime_enabled(smmu->dev))
2188 pm_runtime_force_suspend(smmu->dev);
2190 clk_bulk_disable(smmu->num_clks, smmu->clks);
2192 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2195 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2197 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2200 ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2204 arm_smmu_device_reset(smmu);
2209 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2211 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2213 clk_bulk_disable(smmu->num_clks, smmu->clks);
2218 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2220 if (pm_runtime_suspended(dev))
2223 return arm_smmu_runtime_resume(dev);
2226 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2228 if (pm_runtime_suspended(dev))
2231 return arm_smmu_runtime_suspend(dev);
2234 static const struct dev_pm_ops arm_smmu_pm_ops = {
2235 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2236 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2237 arm_smmu_runtime_resume, NULL)
2240 static struct platform_driver arm_smmu_driver = {
2243 .of_match_table = of_match_ptr(arm_smmu_of_match),
2244 .pm = &arm_smmu_pm_ops,
2245 .suppress_bind_attrs = true,
2247 .probe = arm_smmu_device_probe,
2248 .shutdown = arm_smmu_device_shutdown,
2250 builtin_platform_driver(arm_smmu_driver);