2 * IOMMU API for ARM architected SMMU implementations.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 * Copyright (C) 2013 ARM Limited
19 * Author: Will Deacon <will.deacon@arm.com>
21 * This driver currently supports:
22 * - SMMUv1 and v2 implementations
23 * - Stream-matching and stream-indexing
24 * - v7/v8 long-descriptor format
25 * - Non-secure access to the SMMU
26 * - Context fault reporting
29 #define pr_fmt(fmt) "arm-smmu: " fmt
31 #include <linux/atomic.h>
32 #include <linux/delay.h>
33 #include <linux/dma-iommu.h>
34 #include <linux/dma-mapping.h>
35 #include <linux/err.h>
36 #include <linux/interrupt.h>
38 #include <linux/io-64-nonatomic-hi-lo.h>
39 #include <linux/iommu.h>
40 #include <linux/iopoll.h>
41 #include <linux/module.h>
43 #include <linux/of_address.h>
44 #include <linux/of_device.h>
45 #include <linux/of_iommu.h>
46 #include <linux/pci.h>
47 #include <linux/platform_device.h>
48 #include <linux/slab.h>
49 #include <linux/spinlock.h>
51 #include <linux/amba/bus.h>
53 #include "io-pgtable.h"
55 /* Maximum number of context banks per SMMU */
56 #define ARM_SMMU_MAX_CBS 128
58 /* SMMU global address space */
59 #define ARM_SMMU_GR0(smmu) ((smmu)->base)
60 #define ARM_SMMU_GR1(smmu) ((smmu)->base + (1 << (smmu)->pgshift))
63 * SMMU global address space with conditional offset to access secure
64 * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
67 #define ARM_SMMU_GR0_NS(smmu) \
69 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) \
73 * Some 64-bit registers only make sense to write atomically, but in such
74 * cases all the data relevant to AArch32 formats lies within the lower word,
75 * therefore this actually makes more sense than it might first appear.
78 #define smmu_write_atomic_lq writeq_relaxed
80 #define smmu_write_atomic_lq writel_relaxed
83 /* Configuration registers */
84 #define ARM_SMMU_GR0_sCR0 0x0
85 #define sCR0_CLIENTPD (1 << 0)
86 #define sCR0_GFRE (1 << 1)
87 #define sCR0_GFIE (1 << 2)
88 #define sCR0_GCFGFRE (1 << 4)
89 #define sCR0_GCFGFIE (1 << 5)
90 #define sCR0_USFCFG (1 << 10)
91 #define sCR0_VMIDPNE (1 << 11)
92 #define sCR0_PTM (1 << 12)
93 #define sCR0_FB (1 << 13)
94 #define sCR0_VMID16EN (1 << 31)
95 #define sCR0_BSU_SHIFT 14
96 #define sCR0_BSU_MASK 0x3
98 /* Auxiliary Configuration register */
99 #define ARM_SMMU_GR0_sACR 0x10
101 /* Identification registers */
102 #define ARM_SMMU_GR0_ID0 0x20
103 #define ARM_SMMU_GR0_ID1 0x24
104 #define ARM_SMMU_GR0_ID2 0x28
105 #define ARM_SMMU_GR0_ID3 0x2c
106 #define ARM_SMMU_GR0_ID4 0x30
107 #define ARM_SMMU_GR0_ID5 0x34
108 #define ARM_SMMU_GR0_ID6 0x38
109 #define ARM_SMMU_GR0_ID7 0x3c
110 #define ARM_SMMU_GR0_sGFSR 0x48
111 #define ARM_SMMU_GR0_sGFSYNR0 0x50
112 #define ARM_SMMU_GR0_sGFSYNR1 0x54
113 #define ARM_SMMU_GR0_sGFSYNR2 0x58
115 #define ID0_S1TS (1 << 30)
116 #define ID0_S2TS (1 << 29)
117 #define ID0_NTS (1 << 28)
118 #define ID0_SMS (1 << 27)
119 #define ID0_ATOSNS (1 << 26)
120 #define ID0_PTFS_NO_AARCH32 (1 << 25)
121 #define ID0_PTFS_NO_AARCH32S (1 << 24)
122 #define ID0_CTTW (1 << 14)
123 #define ID0_NUMIRPT_SHIFT 16
124 #define ID0_NUMIRPT_MASK 0xff
125 #define ID0_NUMSIDB_SHIFT 9
126 #define ID0_NUMSIDB_MASK 0xf
127 #define ID0_NUMSMRG_SHIFT 0
128 #define ID0_NUMSMRG_MASK 0xff
130 #define ID1_PAGESIZE (1 << 31)
131 #define ID1_NUMPAGENDXB_SHIFT 28
132 #define ID1_NUMPAGENDXB_MASK 7
133 #define ID1_NUMS2CB_SHIFT 16
134 #define ID1_NUMS2CB_MASK 0xff
135 #define ID1_NUMCB_SHIFT 0
136 #define ID1_NUMCB_MASK 0xff
138 #define ID2_OAS_SHIFT 4
139 #define ID2_OAS_MASK 0xf
140 #define ID2_IAS_SHIFT 0
141 #define ID2_IAS_MASK 0xf
142 #define ID2_UBS_SHIFT 8
143 #define ID2_UBS_MASK 0xf
144 #define ID2_PTFS_4K (1 << 12)
145 #define ID2_PTFS_16K (1 << 13)
146 #define ID2_PTFS_64K (1 << 14)
147 #define ID2_VMID16 (1 << 15)
149 #define ID7_MAJOR_SHIFT 4
150 #define ID7_MAJOR_MASK 0xf
152 /* Global TLB invalidation */
153 #define ARM_SMMU_GR0_TLBIVMID 0x64
154 #define ARM_SMMU_GR0_TLBIALLNSNH 0x68
155 #define ARM_SMMU_GR0_TLBIALLH 0x6c
156 #define ARM_SMMU_GR0_sTLBGSYNC 0x70
157 #define ARM_SMMU_GR0_sTLBGSTATUS 0x74
158 #define sTLBGSTATUS_GSACTIVE (1 << 0)
159 #define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
161 /* Stream mapping registers */
162 #define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
163 #define SMR_VALID (1 << 31)
164 #define SMR_MASK_SHIFT 16
165 #define SMR_ID_SHIFT 0
167 #define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))
168 #define S2CR_CBNDX_SHIFT 0
169 #define S2CR_CBNDX_MASK 0xff
170 #define S2CR_TYPE_SHIFT 16
171 #define S2CR_TYPE_MASK 0x3
172 enum arm_smmu_s2cr_type {
178 #define S2CR_PRIVCFG_SHIFT 24
179 #define S2CR_PRIVCFG_MASK 0x3
180 enum arm_smmu_s2cr_privcfg {
181 S2CR_PRIVCFG_DEFAULT,
187 /* Context bank attribute registers */
188 #define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
189 #define CBAR_VMID_SHIFT 0
190 #define CBAR_VMID_MASK 0xff
191 #define CBAR_S1_BPSHCFG_SHIFT 8
192 #define CBAR_S1_BPSHCFG_MASK 3
193 #define CBAR_S1_BPSHCFG_NSH 3
194 #define CBAR_S1_MEMATTR_SHIFT 12
195 #define CBAR_S1_MEMATTR_MASK 0xf
196 #define CBAR_S1_MEMATTR_WB 0xf
197 #define CBAR_TYPE_SHIFT 16
198 #define CBAR_TYPE_MASK 0x3
199 #define CBAR_TYPE_S2_TRANS (0 << CBAR_TYPE_SHIFT)
200 #define CBAR_TYPE_S1_TRANS_S2_BYPASS (1 << CBAR_TYPE_SHIFT)
201 #define CBAR_TYPE_S1_TRANS_S2_FAULT (2 << CBAR_TYPE_SHIFT)
202 #define CBAR_TYPE_S1_TRANS_S2_TRANS (3 << CBAR_TYPE_SHIFT)
203 #define CBAR_IRPTNDX_SHIFT 24
204 #define CBAR_IRPTNDX_MASK 0xff
206 #define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
207 #define CBA2R_RW64_32BIT (0 << 0)
208 #define CBA2R_RW64_64BIT (1 << 0)
209 #define CBA2R_VMID_SHIFT 16
210 #define CBA2R_VMID_MASK 0xffff
212 /* Translation context bank */
213 #define ARM_SMMU_CB_BASE(smmu) ((smmu)->base + ((smmu)->size >> 1))
214 #define ARM_SMMU_CB(smmu, n) ((n) * (1 << (smmu)->pgshift))
216 #define ARM_SMMU_CB_SCTLR 0x0
217 #define ARM_SMMU_CB_ACTLR 0x4
218 #define ARM_SMMU_CB_RESUME 0x8
219 #define ARM_SMMU_CB_TTBCR2 0x10
220 #define ARM_SMMU_CB_TTBR0 0x20
221 #define ARM_SMMU_CB_TTBR1 0x28
222 #define ARM_SMMU_CB_TTBCR 0x30
223 #define ARM_SMMU_CB_CONTEXTIDR 0x34
224 #define ARM_SMMU_CB_S1_MAIR0 0x38
225 #define ARM_SMMU_CB_S1_MAIR1 0x3c
226 #define ARM_SMMU_CB_PAR 0x50
227 #define ARM_SMMU_CB_FSR 0x58
228 #define ARM_SMMU_CB_FAR 0x60
229 #define ARM_SMMU_CB_FSYNR0 0x68
230 #define ARM_SMMU_CB_S1_TLBIVA 0x600
231 #define ARM_SMMU_CB_S1_TLBIASID 0x610
232 #define ARM_SMMU_CB_S1_TLBIVAL 0x620
233 #define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
234 #define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
235 #define ARM_SMMU_CB_ATS1PR 0x800
236 #define ARM_SMMU_CB_ATSR 0x8f0
238 #define SCTLR_S1_ASIDPNE (1 << 12)
239 #define SCTLR_CFCFG (1 << 7)
240 #define SCTLR_CFIE (1 << 6)
241 #define SCTLR_CFRE (1 << 5)
242 #define SCTLR_E (1 << 4)
243 #define SCTLR_AFE (1 << 2)
244 #define SCTLR_TRE (1 << 1)
245 #define SCTLR_M (1 << 0)
247 #define ARM_MMU500_ACTLR_CPRE (1 << 1)
249 #define ARM_MMU500_ACR_CACHE_LOCK (1 << 26)
251 #define CB_PAR_F (1 << 0)
253 #define ATSR_ACTIVE (1 << 0)
255 #define RESUME_RETRY (0 << 0)
256 #define RESUME_TERMINATE (1 << 0)
258 #define TTBCR2_SEP_SHIFT 15
259 #define TTBCR2_SEP_UPSTREAM (0x7 << TTBCR2_SEP_SHIFT)
261 #define TTBRn_ASID_SHIFT 48
263 #define FSR_MULTI (1 << 31)
264 #define FSR_SS (1 << 30)
265 #define FSR_UUT (1 << 8)
266 #define FSR_ASF (1 << 7)
267 #define FSR_TLBLKF (1 << 6)
268 #define FSR_TLBMCF (1 << 5)
269 #define FSR_EF (1 << 4)
270 #define FSR_PF (1 << 3)
271 #define FSR_AFF (1 << 2)
272 #define FSR_TF (1 << 1)
274 #define FSR_IGN (FSR_AFF | FSR_ASF | \
275 FSR_TLBMCF | FSR_TLBLKF)
276 #define FSR_FAULT (FSR_MULTI | FSR_SS | FSR_UUT | \
277 FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
279 #define FSYNR0_WNR (1 << 4)
281 static int force_stage;
282 module_param(force_stage, int, S_IRUGO);
283 MODULE_PARM_DESC(force_stage,
284 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
285 static bool disable_bypass;
286 module_param(disable_bypass, bool, S_IRUGO);
287 MODULE_PARM_DESC(disable_bypass,
288 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
290 enum arm_smmu_arch_version {
296 enum arm_smmu_implementation {
303 struct arm_smmu_s2cr {
304 struct iommu_group *group;
306 enum arm_smmu_s2cr_type type;
307 enum arm_smmu_s2cr_privcfg privcfg;
311 #define s2cr_init_val (struct arm_smmu_s2cr){ \
312 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
315 struct arm_smmu_smr {
321 struct arm_smmu_master_cfg {
322 struct arm_smmu_device *smmu;
325 #define INVALID_SMENDX -1
326 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
327 #define fwspec_smmu(fw) (__fwspec_cfg(fw)->smmu)
328 #define fwspec_smendx(fw, i) \
329 (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
330 #define for_each_cfg_sme(fw, i, idx) \
331 for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
333 struct arm_smmu_device {
338 unsigned long pgshift;
340 #define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0)
341 #define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1)
342 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 2)
343 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 3)
344 #define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4)
345 #define ARM_SMMU_FEAT_TRANS_OPS (1 << 5)
346 #define ARM_SMMU_FEAT_VMID16 (1 << 6)
347 #define ARM_SMMU_FEAT_FMT_AARCH64_4K (1 << 7)
348 #define ARM_SMMU_FEAT_FMT_AARCH64_16K (1 << 8)
349 #define ARM_SMMU_FEAT_FMT_AARCH64_64K (1 << 9)
350 #define ARM_SMMU_FEAT_FMT_AARCH32_L (1 << 10)
351 #define ARM_SMMU_FEAT_FMT_AARCH32_S (1 << 11)
354 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
356 enum arm_smmu_arch_version version;
357 enum arm_smmu_implementation model;
359 u32 num_context_banks;
360 u32 num_s2_context_banks;
361 DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
364 u32 num_mapping_groups;
367 struct arm_smmu_smr *smrs;
368 struct arm_smmu_s2cr *s2crs;
369 struct mutex stream_map_mutex;
371 unsigned long va_size;
372 unsigned long ipa_size;
373 unsigned long pa_size;
374 unsigned long pgsize_bitmap;
377 u32 num_context_irqs;
380 u32 cavium_id_base; /* Specific to Cavium */
383 enum arm_smmu_context_fmt {
384 ARM_SMMU_CTX_FMT_NONE,
385 ARM_SMMU_CTX_FMT_AARCH64,
386 ARM_SMMU_CTX_FMT_AARCH32_L,
387 ARM_SMMU_CTX_FMT_AARCH32_S,
390 struct arm_smmu_cfg {
394 enum arm_smmu_context_fmt fmt;
396 #define INVALID_IRPTNDX 0xff
398 #define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx)
399 #define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1)
401 enum arm_smmu_domain_stage {
402 ARM_SMMU_DOMAIN_S1 = 0,
404 ARM_SMMU_DOMAIN_NESTED,
407 struct arm_smmu_domain {
408 struct arm_smmu_device *smmu;
409 struct io_pgtable_ops *pgtbl_ops;
410 spinlock_t pgtbl_lock;
411 struct arm_smmu_cfg cfg;
412 enum arm_smmu_domain_stage stage;
413 struct mutex init_mutex; /* Protects smmu pointer */
414 struct iommu_domain domain;
417 struct arm_smmu_option_prop {
422 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
424 static bool using_legacy_binding, using_generic_binding;
426 static struct arm_smmu_option_prop arm_smmu_options[] = {
427 { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
431 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
433 return container_of(dom, struct arm_smmu_domain, domain);
436 static void parse_driver_options(struct arm_smmu_device *smmu)
441 if (of_property_read_bool(smmu->dev->of_node,
442 arm_smmu_options[i].prop)) {
443 smmu->options |= arm_smmu_options[i].opt;
444 dev_notice(smmu->dev, "option %s\n",
445 arm_smmu_options[i].prop);
447 } while (arm_smmu_options[++i].opt);
450 static struct device_node *dev_get_dev_node(struct device *dev)
452 if (dev_is_pci(dev)) {
453 struct pci_bus *bus = to_pci_dev(dev)->bus;
455 while (!pci_is_root_bus(bus))
457 return of_node_get(bus->bridge->parent->of_node);
460 return of_node_get(dev->of_node);
463 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
465 *((__be32 *)data) = cpu_to_be32(alias);
466 return 0; /* Continue walking */
469 static int __find_legacy_master_phandle(struct device *dev, void *data)
471 struct of_phandle_iterator *it = *(void **)data;
472 struct device_node *np = it->node;
475 of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
476 "#stream-id-cells", 0)
477 if (it->node == np) {
478 *(void **)data = dev;
482 return err == -ENOENT ? 0 : err;
485 static struct platform_driver arm_smmu_driver;
486 static struct iommu_ops arm_smmu_ops;
488 static int arm_smmu_register_legacy_master(struct device *dev,
489 struct arm_smmu_device **smmu)
491 struct device *smmu_dev;
492 struct device_node *np;
493 struct of_phandle_iterator it;
499 np = dev_get_dev_node(dev);
500 if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
506 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
507 __find_legacy_master_phandle);
515 if (dev_is_pci(dev)) {
516 /* "mmu-masters" assumes Stream ID == Requester ID */
517 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
523 err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
528 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
532 *smmu = dev_get_drvdata(smmu_dev);
533 of_phandle_iterator_args(&it, sids, it.cur_count);
534 err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
539 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
544 idx = find_next_zero_bit(map, end, start);
547 } while (test_and_set_bit(idx, map));
552 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
557 /* Wait for any pending TLB invalidations to complete */
558 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
561 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
563 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC);
564 while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS)
565 & sTLBGSTATUS_GSACTIVE) {
567 if (++count == TLB_LOOP_TIMEOUT) {
568 dev_err_ratelimited(smmu->dev,
569 "TLB sync timed out -- SMMU may be deadlocked\n");
576 static void arm_smmu_tlb_sync(void *cookie)
578 struct arm_smmu_domain *smmu_domain = cookie;
579 __arm_smmu_tlb_sync(smmu_domain->smmu);
582 static void arm_smmu_tlb_inv_context(void *cookie)
584 struct arm_smmu_domain *smmu_domain = cookie;
585 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
586 struct arm_smmu_device *smmu = smmu_domain->smmu;
587 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
591 base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
592 writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg),
593 base + ARM_SMMU_CB_S1_TLBIASID);
595 base = ARM_SMMU_GR0(smmu);
596 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg),
597 base + ARM_SMMU_GR0_TLBIVMID);
600 __arm_smmu_tlb_sync(smmu);
603 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
604 size_t granule, bool leaf, void *cookie)
606 struct arm_smmu_domain *smmu_domain = cookie;
607 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
608 struct arm_smmu_device *smmu = smmu_domain->smmu;
609 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
613 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
614 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
616 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
618 iova |= ARM_SMMU_CB_ASID(smmu, cfg);
620 writel_relaxed(iova, reg);
622 } while (size -= granule);
625 iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48;
627 writeq_relaxed(iova, reg);
628 iova += granule >> 12;
629 } while (size -= granule);
631 } else if (smmu->version == ARM_SMMU_V2) {
632 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
633 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
634 ARM_SMMU_CB_S2_TLBIIPAS2;
637 smmu_write_atomic_lq(iova, reg);
638 iova += granule >> 12;
639 } while (size -= granule);
641 reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
642 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg);
646 static struct iommu_gather_ops arm_smmu_gather_ops = {
647 .tlb_flush_all = arm_smmu_tlb_inv_context,
648 .tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
649 .tlb_sync = arm_smmu_tlb_sync,
652 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
656 struct iommu_domain *domain = dev;
657 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
658 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
659 struct arm_smmu_device *smmu = smmu_domain->smmu;
660 void __iomem *cb_base;
662 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
663 fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
665 if (!(fsr & FSR_FAULT))
668 fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
669 iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
671 dev_err_ratelimited(smmu->dev,
672 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n",
673 fsr, iova, fsynr, cfg->cbndx);
675 writel(fsr, cb_base + ARM_SMMU_CB_FSR);
679 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
681 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
682 struct arm_smmu_device *smmu = dev;
683 void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
685 gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
686 gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
687 gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
688 gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
693 dev_err_ratelimited(smmu->dev,
694 "Unexpected global fault, this could be serious\n");
695 dev_err_ratelimited(smmu->dev,
696 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
697 gfsr, gfsynr0, gfsynr1, gfsynr2);
699 writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
703 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
704 struct io_pgtable_cfg *pgtbl_cfg)
709 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
710 struct arm_smmu_device *smmu = smmu_domain->smmu;
711 void __iomem *cb_base, *gr1_base;
713 gr1_base = ARM_SMMU_GR1(smmu);
714 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
715 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
717 if (smmu->version > ARM_SMMU_V1) {
718 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
719 reg = CBA2R_RW64_64BIT;
721 reg = CBA2R_RW64_32BIT;
722 /* 16-bit VMIDs live in CBA2R */
723 if (smmu->features & ARM_SMMU_FEAT_VMID16)
724 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT;
726 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
731 if (smmu->version < ARM_SMMU_V2)
732 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
735 * Use the weakest shareability/memory types, so they are
736 * overridden by the ttbcr/pte.
739 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
740 (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
741 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
742 /* 8-bit VMIDs live in CBAR */
743 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT;
745 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
749 u16 asid = ARM_SMMU_CB_ASID(smmu, cfg);
751 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
752 reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
753 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0);
754 reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
755 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1);
756 writel_relaxed(asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
758 reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
759 reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
760 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
761 reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
762 reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
763 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
766 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
767 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
772 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
773 reg = pgtbl_cfg->arm_v7s_cfg.tcr;
776 reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
777 reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
778 reg2 |= TTBCR2_SEP_UPSTREAM;
780 if (smmu->version > ARM_SMMU_V1)
781 writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2);
783 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
785 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
787 /* MAIRs (stage-1 only) */
789 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
790 reg = pgtbl_cfg->arm_v7s_cfg.prrr;
791 reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr;
793 reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
794 reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
796 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
797 writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1);
801 reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
803 reg |= SCTLR_S1_ASIDPNE;
807 writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
810 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
811 struct arm_smmu_device *smmu)
813 int irq, start, ret = 0;
814 unsigned long ias, oas;
815 struct io_pgtable_ops *pgtbl_ops;
816 struct io_pgtable_cfg pgtbl_cfg;
817 enum io_pgtable_fmt fmt;
818 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
819 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
821 mutex_lock(&smmu_domain->init_mutex);
822 if (smmu_domain->smmu)
826 * Mapping the requested stage onto what we support is surprisingly
827 * complicated, mainly because the spec allows S1+S2 SMMUs without
828 * support for nested translation. That means we end up with the
831 * Requested Supported Actual
841 * Note that you can't actually request stage-2 mappings.
843 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
844 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
845 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
846 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
849 * Choosing a suitable context format is even more fiddly. Until we
850 * grow some way for the caller to express a preference, and/or move
851 * the decision into the io-pgtable code where it arguably belongs,
852 * just aim for the closest thing to the rest of the system, and hope
853 * that the hardware isn't esoteric enough that we can't assume AArch64
854 * support to be a superset of AArch32 support...
856 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
857 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
858 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
859 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
860 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
861 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
862 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
863 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
864 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
865 ARM_SMMU_FEAT_FMT_AARCH64_16K |
866 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
867 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
869 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
874 switch (smmu_domain->stage) {
875 case ARM_SMMU_DOMAIN_S1:
876 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
877 start = smmu->num_s2_context_banks;
879 oas = smmu->ipa_size;
880 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
881 fmt = ARM_64_LPAE_S1;
882 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
883 fmt = ARM_32_LPAE_S1;
884 ias = min(ias, 32UL);
885 oas = min(oas, 40UL);
888 ias = min(ias, 32UL);
889 oas = min(oas, 32UL);
892 case ARM_SMMU_DOMAIN_NESTED:
894 * We will likely want to change this if/when KVM gets
897 case ARM_SMMU_DOMAIN_S2:
898 cfg->cbar = CBAR_TYPE_S2_TRANS;
900 ias = smmu->ipa_size;
902 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
903 fmt = ARM_64_LPAE_S2;
905 fmt = ARM_32_LPAE_S2;
906 ias = min(ias, 40UL);
907 oas = min(oas, 40UL);
915 ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
916 smmu->num_context_banks);
921 if (smmu->version < ARM_SMMU_V2) {
922 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
923 cfg->irptndx %= smmu->num_context_irqs;
925 cfg->irptndx = cfg->cbndx;
928 pgtbl_cfg = (struct io_pgtable_cfg) {
929 .pgsize_bitmap = smmu->pgsize_bitmap,
932 .tlb = &arm_smmu_gather_ops,
933 .iommu_dev = smmu->dev,
936 smmu_domain->smmu = smmu;
937 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
943 /* Update the domain's page sizes to reflect the page table format */
944 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
945 domain->geometry.aperture_end = (1UL << ias) - 1;
946 domain->geometry.force_aperture = true;
948 /* Initialise the context bank with our page table cfg */
949 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
952 * Request context fault interrupt. Do this last to avoid the
953 * handler seeing a half-initialised domain state.
955 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
956 ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
957 IRQF_SHARED, "arm-smmu-context-fault", domain);
959 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
961 cfg->irptndx = INVALID_IRPTNDX;
964 mutex_unlock(&smmu_domain->init_mutex);
966 /* Publish page table ops for map/unmap */
967 smmu_domain->pgtbl_ops = pgtbl_ops;
971 smmu_domain->smmu = NULL;
973 mutex_unlock(&smmu_domain->init_mutex);
977 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
979 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
980 struct arm_smmu_device *smmu = smmu_domain->smmu;
981 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
982 void __iomem *cb_base;
989 * Disable the context bank and free the page tables before freeing
992 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
993 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
995 if (cfg->irptndx != INVALID_IRPTNDX) {
996 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
997 devm_free_irq(smmu->dev, irq, domain);
1000 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1001 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
1004 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1006 struct arm_smmu_domain *smmu_domain;
1008 if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
1011 * Allocate the domain and initialise some of its data structures.
1012 * We can't really do anything meaningful until we've added a
1015 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1019 if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
1020 iommu_get_dma_cookie(&smmu_domain->domain))) {
1025 mutex_init(&smmu_domain->init_mutex);
1026 spin_lock_init(&smmu_domain->pgtbl_lock);
1028 return &smmu_domain->domain;
1031 static void arm_smmu_domain_free(struct iommu_domain *domain)
1033 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1036 * Free the domain resources. We assume that all devices have
1037 * already been detached.
1039 iommu_put_dma_cookie(domain);
1040 arm_smmu_destroy_domain_context(domain);
1044 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
1046 struct arm_smmu_smr *smr = smmu->smrs + idx;
1047 u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT;
1051 writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
1054 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
1056 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
1057 u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT |
1058 (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT |
1059 (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT;
1061 writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
1064 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
1066 arm_smmu_write_s2cr(smmu, idx);
1068 arm_smmu_write_smr(smmu, idx);
1071 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1073 struct arm_smmu_smr *smrs = smmu->smrs;
1074 int i, free_idx = -ENOSPC;
1076 /* Stream indexing is blissfully easy */
1080 /* Validating SMRs is... less so */
1081 for (i = 0; i < smmu->num_mapping_groups; ++i) {
1082 if (!smrs[i].valid) {
1084 * Note the first free entry we come across, which
1085 * we'll claim in the end if nothing else matches.
1092 * If the new entry is _entirely_ matched by an existing entry,
1093 * then reuse that, with the guarantee that there also cannot
1094 * be any subsequent conflicting entries. In normal use we'd
1095 * expect simply identical entries for this case, but there's
1096 * no harm in accommodating the generalisation.
1098 if ((mask & smrs[i].mask) == mask &&
1099 !((id ^ smrs[i].id) & ~smrs[i].mask))
1102 * If the new entry has any other overlap with an existing one,
1103 * though, then there always exists at least one stream ID
1104 * which would cause a conflict, and we can't allow that risk.
1106 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1113 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1115 if (--smmu->s2crs[idx].count)
1118 smmu->s2crs[idx] = s2cr_init_val;
1120 smmu->smrs[idx].valid = false;
1125 static int arm_smmu_master_alloc_smes(struct device *dev)
1127 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1128 struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1129 struct arm_smmu_device *smmu = cfg->smmu;
1130 struct arm_smmu_smr *smrs = smmu->smrs;
1131 struct iommu_group *group;
1134 mutex_lock(&smmu->stream_map_mutex);
1135 /* Figure out a viable stream map entry allocation */
1136 for_each_cfg_sme(fwspec, i, idx) {
1137 u16 sid = fwspec->ids[i];
1138 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1140 if (idx != INVALID_SMENDX) {
1145 ret = arm_smmu_find_sme(smmu, sid, mask);
1150 if (smrs && smmu->s2crs[idx].count == 0) {
1152 smrs[idx].mask = mask;
1153 smrs[idx].valid = true;
1155 smmu->s2crs[idx].count++;
1156 cfg->smendx[i] = (s16)idx;
1159 group = iommu_group_get_for_dev(dev);
1161 group = ERR_PTR(-ENOMEM);
1162 if (IS_ERR(group)) {
1163 ret = PTR_ERR(group);
1166 iommu_group_put(group);
1168 /* It worked! Now, poke the actual hardware */
1169 for_each_cfg_sme(fwspec, i, idx) {
1170 arm_smmu_write_sme(smmu, idx);
1171 smmu->s2crs[idx].group = group;
1174 mutex_unlock(&smmu->stream_map_mutex);
1179 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1180 cfg->smendx[i] = INVALID_SMENDX;
1182 mutex_unlock(&smmu->stream_map_mutex);
1186 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1188 struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1189 struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1192 mutex_lock(&smmu->stream_map_mutex);
1193 for_each_cfg_sme(fwspec, i, idx) {
1194 if (arm_smmu_free_sme(smmu, idx))
1195 arm_smmu_write_sme(smmu, idx);
1196 cfg->smendx[i] = INVALID_SMENDX;
1198 mutex_unlock(&smmu->stream_map_mutex);
1201 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1202 struct iommu_fwspec *fwspec)
1204 struct arm_smmu_device *smmu = smmu_domain->smmu;
1205 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1206 enum arm_smmu_s2cr_type type = S2CR_TYPE_TRANS;
1207 u8 cbndx = smmu_domain->cfg.cbndx;
1210 for_each_cfg_sme(fwspec, i, idx) {
1211 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1214 s2cr[idx].type = type;
1215 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1216 s2cr[idx].cbndx = cbndx;
1217 arm_smmu_write_s2cr(smmu, idx);
1222 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1225 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1226 struct arm_smmu_device *smmu;
1227 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1229 if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1230 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1235 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1236 * domains between of_xlate() and add_device() - we have no way to cope
1237 * with that, so until ARM gets converted to rely on groups and default
1238 * domains, just say no (but more politely than by dereferencing NULL).
1239 * This should be at least a WARN_ON once that's sorted.
1241 if (!fwspec->iommu_priv)
1244 smmu = fwspec_smmu(fwspec);
1245 /* Ensure that the domain is finalised */
1246 ret = arm_smmu_init_domain_context(domain, smmu);
1251 * Sanity check the domain. We don't support domains across
1254 if (smmu_domain->smmu != smmu) {
1256 "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1257 dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1261 /* Looks ok, so add the device to the domain */
1262 return arm_smmu_domain_add_master(smmu_domain, fwspec);
1265 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1266 phys_addr_t paddr, size_t size, int prot)
1269 unsigned long flags;
1270 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1271 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1276 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1277 ret = ops->map(ops, iova, paddr, size, prot);
1278 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1282 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1286 unsigned long flags;
1287 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1288 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1293 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1294 ret = ops->unmap(ops, iova, size);
1295 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1299 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1302 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1303 struct arm_smmu_device *smmu = smmu_domain->smmu;
1304 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1305 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1306 struct device *dev = smmu->dev;
1307 void __iomem *cb_base;
1312 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1314 /* ATS1 registers can only be written atomically */
1315 va = iova & ~0xfffUL;
1316 if (smmu->version == ARM_SMMU_V2)
1317 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1318 else /* Register is only 32-bit in v1 */
1319 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1321 if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1322 !(tmp & ATSR_ACTIVE), 5, 50)) {
1324 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1326 return ops->iova_to_phys(ops, iova);
1329 phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1330 if (phys & CB_PAR_F) {
1331 dev_err(dev, "translation fault!\n");
1332 dev_err(dev, "PAR = 0x%llx\n", phys);
1336 return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1339 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1343 unsigned long flags;
1344 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1345 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1350 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1351 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1352 smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1353 ret = arm_smmu_iova_to_phys_hard(domain, iova);
1355 ret = ops->iova_to_phys(ops, iova);
1358 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1363 static bool arm_smmu_capable(enum iommu_cap cap)
1366 case IOMMU_CAP_CACHE_COHERENCY:
1368 * Return true here as the SMMU can always send out coherent
1372 case IOMMU_CAP_INTR_REMAP:
1373 return true; /* MSIs are just memory writes */
1374 case IOMMU_CAP_NOEXEC:
1381 static int arm_smmu_match_node(struct device *dev, void *data)
1383 return dev->of_node == data;
1386 static struct arm_smmu_device *arm_smmu_get_by_node(struct device_node *np)
1388 struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1389 np, arm_smmu_match_node);
1391 return dev ? dev_get_drvdata(dev) : NULL;
1394 static int arm_smmu_add_device(struct device *dev)
1396 struct arm_smmu_device *smmu;
1397 struct arm_smmu_master_cfg *cfg;
1398 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1401 if (using_legacy_binding) {
1402 ret = arm_smmu_register_legacy_master(dev, &smmu);
1403 fwspec = dev->iommu_fwspec;
1406 } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1407 smmu = arm_smmu_get_by_node(to_of_node(fwspec->iommu_fwnode));
1413 for (i = 0; i < fwspec->num_ids; i++) {
1414 u16 sid = fwspec->ids[i];
1415 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1417 if (sid & ~smmu->streamid_mask) {
1418 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1419 sid, smmu->streamid_mask);
1422 if (mask & ~smmu->smr_mask_mask) {
1423 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1424 sid, smmu->smr_mask_mask);
1430 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1436 fwspec->iommu_priv = cfg;
1438 cfg->smendx[i] = INVALID_SMENDX;
1440 ret = arm_smmu_master_alloc_smes(dev);
1448 kfree(fwspec->iommu_priv);
1449 iommu_fwspec_free(dev);
1453 static void arm_smmu_remove_device(struct device *dev)
1455 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1457 if (!fwspec || fwspec->ops != &arm_smmu_ops)
1460 arm_smmu_master_free_smes(fwspec);
1461 iommu_group_remove_device(dev);
1462 kfree(fwspec->iommu_priv);
1463 iommu_fwspec_free(dev);
1466 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1468 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1469 struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1470 struct iommu_group *group = NULL;
1473 for_each_cfg_sme(fwspec, i, idx) {
1474 if (group && smmu->s2crs[idx].group &&
1475 group != smmu->s2crs[idx].group)
1476 return ERR_PTR(-EINVAL);
1478 group = smmu->s2crs[idx].group;
1484 if (dev_is_pci(dev))
1485 group = pci_device_group(dev);
1487 group = generic_device_group(dev);
1492 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1493 enum iommu_attr attr, void *data)
1495 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1498 case DOMAIN_ATTR_NESTING:
1499 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1506 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1507 enum iommu_attr attr, void *data)
1510 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1512 mutex_lock(&smmu_domain->init_mutex);
1515 case DOMAIN_ATTR_NESTING:
1516 if (smmu_domain->smmu) {
1522 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1524 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1532 mutex_unlock(&smmu_domain->init_mutex);
1536 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1540 if (args->args_count > 0)
1541 fwid |= (u16)args->args[0];
1543 if (args->args_count > 1)
1544 fwid |= (u16)args->args[1] << SMR_MASK_SHIFT;
1546 return iommu_fwspec_add_ids(dev, &fwid, 1);
1549 static struct iommu_ops arm_smmu_ops = {
1550 .capable = arm_smmu_capable,
1551 .domain_alloc = arm_smmu_domain_alloc,
1552 .domain_free = arm_smmu_domain_free,
1553 .attach_dev = arm_smmu_attach_dev,
1554 .map = arm_smmu_map,
1555 .unmap = arm_smmu_unmap,
1556 .map_sg = default_iommu_map_sg,
1557 .iova_to_phys = arm_smmu_iova_to_phys,
1558 .add_device = arm_smmu_add_device,
1559 .remove_device = arm_smmu_remove_device,
1560 .device_group = arm_smmu_device_group,
1561 .domain_get_attr = arm_smmu_domain_get_attr,
1562 .domain_set_attr = arm_smmu_domain_set_attr,
1563 .of_xlate = arm_smmu_of_xlate,
1564 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1567 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1569 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1570 void __iomem *cb_base;
1574 /* clear global FSR */
1575 reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1576 writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1579 * Reset stream mapping groups: Initial values mark all SMRn as
1580 * invalid and all S2CRn as bypass unless overridden.
1582 for (i = 0; i < smmu->num_mapping_groups; ++i)
1583 arm_smmu_write_sme(smmu, i);
1586 * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1587 * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1588 * bit is only present in MMU-500r2 onwards.
1590 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1591 major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
1592 if ((smmu->model == ARM_MMU500) && (major >= 2)) {
1593 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1594 reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1595 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1598 /* Make sure all context banks are disabled and clear CB_FSR */
1599 for (i = 0; i < smmu->num_context_banks; ++i) {
1600 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
1601 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1602 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1604 * Disable MMU-500's not-particularly-beneficial next-page
1605 * prefetcher for the sake of errata #841119 and #826419.
1607 if (smmu->model == ARM_MMU500) {
1608 reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1609 reg &= ~ARM_MMU500_ACTLR_CPRE;
1610 writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1614 /* Invalidate the TLB, just in case */
1615 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1616 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1618 reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1620 /* Enable fault reporting */
1621 reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1623 /* Disable TLB broadcasting. */
1624 reg |= (sCR0_VMIDPNE | sCR0_PTM);
1626 /* Enable client access, handling unmatched streams as appropriate */
1627 reg &= ~sCR0_CLIENTPD;
1631 reg &= ~sCR0_USFCFG;
1633 /* Disable forced broadcasting */
1636 /* Don't upgrade barriers */
1637 reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1639 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1640 reg |= sCR0_VMID16EN;
1642 /* Push the button */
1643 __arm_smmu_tlb_sync(smmu);
1644 writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1647 static int arm_smmu_id_size_to_bits(int size)
1666 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1669 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1671 bool cttw_dt, cttw_reg;
1674 dev_notice(smmu->dev, "probing hardware configuration...\n");
1675 dev_notice(smmu->dev, "SMMUv%d with:\n",
1676 smmu->version == ARM_SMMU_V2 ? 2 : 1);
1679 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1681 /* Restrict available stages based on module parameter */
1682 if (force_stage == 1)
1683 id &= ~(ID0_S2TS | ID0_NTS);
1684 else if (force_stage == 2)
1685 id &= ~(ID0_S1TS | ID0_NTS);
1687 if (id & ID0_S1TS) {
1688 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1689 dev_notice(smmu->dev, "\tstage 1 translation\n");
1692 if (id & ID0_S2TS) {
1693 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1694 dev_notice(smmu->dev, "\tstage 2 translation\n");
1698 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1699 dev_notice(smmu->dev, "\tnested translation\n");
1702 if (!(smmu->features &
1703 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1704 dev_err(smmu->dev, "\tno translation support!\n");
1708 if ((id & ID0_S1TS) &&
1709 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1710 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1711 dev_notice(smmu->dev, "\taddress translation ops\n");
1715 * In order for DMA API calls to work properly, we must defer to what
1716 * the DT says about coherency, regardless of what the hardware claims.
1717 * Fortunately, this also opens up a workaround for systems where the
1718 * ID register value has ended up configured incorrectly.
1720 cttw_dt = of_dma_is_coherent(smmu->dev->of_node);
1721 cttw_reg = !!(id & ID0_CTTW);
1723 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1724 if (cttw_dt || cttw_reg)
1725 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1726 cttw_dt ? "" : "non-");
1727 if (cttw_dt != cttw_reg)
1728 dev_notice(smmu->dev,
1729 "\t(IDR0.CTTW overridden by dma-coherent property)\n");
1731 /* Max. number of entries we have for stream matching/indexing */
1732 size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK);
1733 smmu->streamid_mask = size - 1;
1737 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1738 size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK;
1741 "stream-matching supported, but no SMRs present!\n");
1746 * SMR.ID bits may not be preserved if the corresponding MASK
1747 * bits are set, so check each one separately. We can reject
1748 * masters later if they try to claim IDs outside these masks.
1750 smr = smmu->streamid_mask << SMR_ID_SHIFT;
1751 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1752 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1753 smmu->streamid_mask = smr >> SMR_ID_SHIFT;
1755 smr = smmu->streamid_mask << SMR_MASK_SHIFT;
1756 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1757 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1758 smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT;
1760 /* Zero-initialised to mark as invalid */
1761 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1766 dev_notice(smmu->dev,
1767 "\tstream matching with %lu register groups, mask 0x%x",
1768 size, smmu->smr_mask_mask);
1770 /* s2cr->type == 0 means translation, so initialise explicitly */
1771 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1775 for (i = 0; i < size; i++)
1776 smmu->s2crs[i] = s2cr_init_val;
1778 smmu->num_mapping_groups = size;
1779 mutex_init(&smmu->stream_map_mutex);
1781 if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1782 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1783 if (!(id & ID0_PTFS_NO_AARCH32S))
1784 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1788 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1789 smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1791 /* Check for size mismatch of SMMU address space from mapped region */
1792 size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1793 size *= 2 << smmu->pgshift;
1794 if (smmu->size != size)
1796 "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1799 smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1800 smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1801 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1802 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1805 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1806 smmu->num_context_banks, smmu->num_s2_context_banks);
1808 * Cavium CN88xx erratum #27704.
1809 * Ensure ASID and VMID allocation is unique across all SMMUs in
1812 if (smmu->model == CAVIUM_SMMUV2) {
1813 smmu->cavium_id_base =
1814 atomic_add_return(smmu->num_context_banks,
1815 &cavium_smmu_context_count);
1816 smmu->cavium_id_base -= smmu->num_context_banks;
1820 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1821 size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1822 smmu->ipa_size = size;
1824 /* The output mask is also applied for bypass */
1825 size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1826 smmu->pa_size = size;
1828 if (id & ID2_VMID16)
1829 smmu->features |= ARM_SMMU_FEAT_VMID16;
1832 * What the page table walker can address actually depends on which
1833 * descriptor format is in use, but since a) we don't know that yet,
1834 * and b) it can vary per context bank, this will have to do...
1836 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1838 "failed to set DMA mask for table walker\n");
1840 if (smmu->version < ARM_SMMU_V2) {
1841 smmu->va_size = smmu->ipa_size;
1842 if (smmu->version == ARM_SMMU_V1_64K)
1843 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1845 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1846 smmu->va_size = arm_smmu_id_size_to_bits(size);
1847 if (id & ID2_PTFS_4K)
1848 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1849 if (id & ID2_PTFS_16K)
1850 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1851 if (id & ID2_PTFS_64K)
1852 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1855 /* Now we've corralled the various formats, what'll it do? */
1856 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1857 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1858 if (smmu->features &
1859 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1860 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1861 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1862 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1863 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1864 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1866 if (arm_smmu_ops.pgsize_bitmap == -1UL)
1867 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1869 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1870 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1871 smmu->pgsize_bitmap);
1874 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1875 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1876 smmu->va_size, smmu->ipa_size);
1878 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1879 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1880 smmu->ipa_size, smmu->pa_size);
1885 struct arm_smmu_match_data {
1886 enum arm_smmu_arch_version version;
1887 enum arm_smmu_implementation model;
1890 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1891 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
1893 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1894 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1895 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1896 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1897 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1898 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1900 static const struct of_device_id arm_smmu_of_match[] = {
1901 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1902 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1903 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1904 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1905 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1906 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1907 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1910 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1912 static int arm_smmu_device_dt_probe(struct platform_device *pdev)
1914 const struct arm_smmu_match_data *data;
1915 struct resource *res;
1916 struct arm_smmu_device *smmu;
1917 struct device *dev = &pdev->dev;
1918 int num_irqs, i, err;
1919 bool legacy_binding;
1921 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
1922 if (legacy_binding && !using_generic_binding) {
1923 if (!using_legacy_binding)
1924 pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
1925 using_legacy_binding = true;
1926 } else if (!legacy_binding && !using_legacy_binding) {
1927 using_generic_binding = true;
1929 dev_err(dev, "not probing due to mismatched DT properties\n");
1933 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
1935 dev_err(dev, "failed to allocate arm_smmu_device\n");
1940 data = of_device_get_match_data(dev);
1941 smmu->version = data->version;
1942 smmu->model = data->model;
1944 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1945 smmu->base = devm_ioremap_resource(dev, res);
1946 if (IS_ERR(smmu->base))
1947 return PTR_ERR(smmu->base);
1948 smmu->size = resource_size(res);
1950 if (of_property_read_u32(dev->of_node, "#global-interrupts",
1951 &smmu->num_global_irqs)) {
1952 dev_err(dev, "missing #global-interrupts property\n");
1957 while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
1959 if (num_irqs > smmu->num_global_irqs)
1960 smmu->num_context_irqs++;
1963 if (!smmu->num_context_irqs) {
1964 dev_err(dev, "found %d interrupts but expected at least %d\n",
1965 num_irqs, smmu->num_global_irqs + 1);
1969 smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
1972 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
1976 for (i = 0; i < num_irqs; ++i) {
1977 int irq = platform_get_irq(pdev, i);
1980 dev_err(dev, "failed to get irq index %d\n", i);
1983 smmu->irqs[i] = irq;
1986 err = arm_smmu_device_cfg_probe(smmu);
1990 parse_driver_options(smmu);
1992 if (smmu->version == ARM_SMMU_V2 &&
1993 smmu->num_context_banks != smmu->num_context_irqs) {
1995 "found only %d context interrupt(s) but %d required\n",
1996 smmu->num_context_irqs, smmu->num_context_banks);
2000 for (i = 0; i < smmu->num_global_irqs; ++i) {
2001 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2002 arm_smmu_global_fault,
2004 "arm-smmu global fault",
2007 dev_err(dev, "failed to request global IRQ %d (%u)\n",
2013 of_iommu_set_ops(dev->of_node, &arm_smmu_ops);
2014 platform_set_drvdata(pdev, smmu);
2015 arm_smmu_device_reset(smmu);
2017 /* Oh, for a proper bus abstraction */
2018 if (!iommu_present(&platform_bus_type))
2019 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2020 #ifdef CONFIG_ARM_AMBA
2021 if (!iommu_present(&amba_bustype))
2022 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2025 if (!iommu_present(&pci_bus_type)) {
2027 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2033 static int arm_smmu_device_remove(struct platform_device *pdev)
2035 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2040 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2041 dev_err(&pdev->dev, "removing device with active domains!\n");
2043 /* Turn the thing off */
2044 writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2048 static struct platform_driver arm_smmu_driver = {
2051 .of_match_table = of_match_ptr(arm_smmu_of_match),
2053 .probe = arm_smmu_device_dt_probe,
2054 .remove = arm_smmu_device_remove,
2057 static int __init arm_smmu_init(void)
2059 static bool registered;
2063 ret = platform_driver_register(&arm_smmu_driver);
2069 static void __exit arm_smmu_exit(void)
2071 return platform_driver_unregister(&arm_smmu_driver);
2074 subsys_initcall(arm_smmu_init);
2075 module_exit(arm_smmu_exit);
2077 static int __init arm_smmu_of_init(struct device_node *np)
2079 int ret = arm_smmu_init();
2084 if (!of_platform_device_create(np, NULL, platform_bus_type.dev_root))
2089 IOMMU_OF_DECLARE(arm_smmuv1, "arm,smmu-v1", arm_smmu_of_init);
2090 IOMMU_OF_DECLARE(arm_smmuv2, "arm,smmu-v2", arm_smmu_of_init);
2091 IOMMU_OF_DECLARE(arm_mmu400, "arm,mmu-400", arm_smmu_of_init);
2092 IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", arm_smmu_of_init);
2093 IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", arm_smmu_of_init);
2094 IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", arm_smmu_of_init);
2096 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2097 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2098 MODULE_LICENSE("GPL v2");