2 * IOMMU API for ARM architected SMMUv3 implementations.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 * Copyright (C) 2015 ARM Limited
18 * Author: Will Deacon <will.deacon@arm.com>
20 * This driver is powered by bad coffee and bombay mix.
23 #include <linux/delay.h>
24 #include <linux/err.h>
25 #include <linux/interrupt.h>
26 #include <linux/iommu.h>
27 #include <linux/iopoll.h>
28 #include <linux/module.h>
29 #include <linux/msi.h>
31 #include <linux/of_address.h>
32 #include <linux/of_platform.h>
33 #include <linux/pci.h>
34 #include <linux/platform_device.h>
36 #include "io-pgtable.h"
39 #define ARM_SMMU_IDR0 0x0
40 #define IDR0_ST_LVL_SHIFT 27
41 #define IDR0_ST_LVL_MASK 0x3
42 #define IDR0_ST_LVL_2LVL (1 << IDR0_ST_LVL_SHIFT)
43 #define IDR0_STALL_MODEL (3 << 24)
44 #define IDR0_TTENDIAN_SHIFT 21
45 #define IDR0_TTENDIAN_MASK 0x3
46 #define IDR0_TTENDIAN_LE (2 << IDR0_TTENDIAN_SHIFT)
47 #define IDR0_TTENDIAN_BE (3 << IDR0_TTENDIAN_SHIFT)
48 #define IDR0_TTENDIAN_MIXED (0 << IDR0_TTENDIAN_SHIFT)
49 #define IDR0_CD2L (1 << 19)
50 #define IDR0_VMID16 (1 << 18)
51 #define IDR0_PRI (1 << 16)
52 #define IDR0_SEV (1 << 14)
53 #define IDR0_MSI (1 << 13)
54 #define IDR0_ASID16 (1 << 12)
55 #define IDR0_ATS (1 << 10)
56 #define IDR0_HYP (1 << 9)
57 #define IDR0_COHACC (1 << 4)
58 #define IDR0_TTF_SHIFT 2
59 #define IDR0_TTF_MASK 0x3
60 #define IDR0_TTF_AARCH64 (2 << IDR0_TTF_SHIFT)
61 #define IDR0_TTF_AARCH32_64 (3 << IDR0_TTF_SHIFT)
62 #define IDR0_S1P (1 << 1)
63 #define IDR0_S2P (1 << 0)
65 #define ARM_SMMU_IDR1 0x4
66 #define IDR1_TABLES_PRESET (1 << 30)
67 #define IDR1_QUEUES_PRESET (1 << 29)
68 #define IDR1_REL (1 << 28)
69 #define IDR1_CMDQ_SHIFT 21
70 #define IDR1_CMDQ_MASK 0x1f
71 #define IDR1_EVTQ_SHIFT 16
72 #define IDR1_EVTQ_MASK 0x1f
73 #define IDR1_PRIQ_SHIFT 11
74 #define IDR1_PRIQ_MASK 0x1f
75 #define IDR1_SSID_SHIFT 6
76 #define IDR1_SSID_MASK 0x1f
77 #define IDR1_SID_SHIFT 0
78 #define IDR1_SID_MASK 0x3f
80 #define ARM_SMMU_IDR5 0x14
81 #define IDR5_STALL_MAX_SHIFT 16
82 #define IDR5_STALL_MAX_MASK 0xffff
83 #define IDR5_GRAN64K (1 << 6)
84 #define IDR5_GRAN16K (1 << 5)
85 #define IDR5_GRAN4K (1 << 4)
86 #define IDR5_OAS_SHIFT 0
87 #define IDR5_OAS_MASK 0x7
88 #define IDR5_OAS_32_BIT (0 << IDR5_OAS_SHIFT)
89 #define IDR5_OAS_36_BIT (1 << IDR5_OAS_SHIFT)
90 #define IDR5_OAS_40_BIT (2 << IDR5_OAS_SHIFT)
91 #define IDR5_OAS_42_BIT (3 << IDR5_OAS_SHIFT)
92 #define IDR5_OAS_44_BIT (4 << IDR5_OAS_SHIFT)
93 #define IDR5_OAS_48_BIT (5 << IDR5_OAS_SHIFT)
95 #define ARM_SMMU_CR0 0x20
96 #define CR0_CMDQEN (1 << 3)
97 #define CR0_EVTQEN (1 << 2)
98 #define CR0_PRIQEN (1 << 1)
99 #define CR0_SMMUEN (1 << 0)
101 #define ARM_SMMU_CR0ACK 0x24
103 #define ARM_SMMU_CR1 0x28
107 #define CR1_CACHE_NC 0
108 #define CR1_CACHE_WB 1
109 #define CR1_CACHE_WT 2
110 #define CR1_TABLE_SH_SHIFT 10
111 #define CR1_TABLE_OC_SHIFT 8
112 #define CR1_TABLE_IC_SHIFT 6
113 #define CR1_QUEUE_SH_SHIFT 4
114 #define CR1_QUEUE_OC_SHIFT 2
115 #define CR1_QUEUE_IC_SHIFT 0
117 #define ARM_SMMU_CR2 0x2c
118 #define CR2_PTM (1 << 2)
119 #define CR2_RECINVSID (1 << 1)
120 #define CR2_E2H (1 << 0)
122 #define ARM_SMMU_IRQ_CTRL 0x50
123 #define IRQ_CTRL_EVTQ_IRQEN (1 << 2)
124 #define IRQ_CTRL_PRIQ_IRQEN (1 << 1)
125 #define IRQ_CTRL_GERROR_IRQEN (1 << 0)
127 #define ARM_SMMU_IRQ_CTRLACK 0x54
129 #define ARM_SMMU_GERROR 0x60
130 #define GERROR_SFM_ERR (1 << 8)
131 #define GERROR_MSI_GERROR_ABT_ERR (1 << 7)
132 #define GERROR_MSI_PRIQ_ABT_ERR (1 << 6)
133 #define GERROR_MSI_EVTQ_ABT_ERR (1 << 5)
134 #define GERROR_MSI_CMDQ_ABT_ERR (1 << 4)
135 #define GERROR_PRIQ_ABT_ERR (1 << 3)
136 #define GERROR_EVTQ_ABT_ERR (1 << 2)
137 #define GERROR_CMDQ_ERR (1 << 0)
138 #define GERROR_ERR_MASK 0xfd
140 #define ARM_SMMU_GERRORN 0x64
142 #define ARM_SMMU_GERROR_IRQ_CFG0 0x68
143 #define ARM_SMMU_GERROR_IRQ_CFG1 0x70
144 #define ARM_SMMU_GERROR_IRQ_CFG2 0x74
146 #define ARM_SMMU_STRTAB_BASE 0x80
147 #define STRTAB_BASE_RA (1UL << 62)
148 #define STRTAB_BASE_ADDR_SHIFT 6
149 #define STRTAB_BASE_ADDR_MASK 0x3ffffffffffUL
151 #define ARM_SMMU_STRTAB_BASE_CFG 0x88
152 #define STRTAB_BASE_CFG_LOG2SIZE_SHIFT 0
153 #define STRTAB_BASE_CFG_LOG2SIZE_MASK 0x3f
154 #define STRTAB_BASE_CFG_SPLIT_SHIFT 6
155 #define STRTAB_BASE_CFG_SPLIT_MASK 0x1f
156 #define STRTAB_BASE_CFG_FMT_SHIFT 16
157 #define STRTAB_BASE_CFG_FMT_MASK 0x3
158 #define STRTAB_BASE_CFG_FMT_LINEAR (0 << STRTAB_BASE_CFG_FMT_SHIFT)
159 #define STRTAB_BASE_CFG_FMT_2LVL (1 << STRTAB_BASE_CFG_FMT_SHIFT)
161 #define ARM_SMMU_CMDQ_BASE 0x90
162 #define ARM_SMMU_CMDQ_PROD 0x98
163 #define ARM_SMMU_CMDQ_CONS 0x9c
165 #define ARM_SMMU_EVTQ_BASE 0xa0
166 #define ARM_SMMU_EVTQ_PROD 0x100a8
167 #define ARM_SMMU_EVTQ_CONS 0x100ac
168 #define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0
169 #define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8
170 #define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc
172 #define ARM_SMMU_PRIQ_BASE 0xc0
173 #define ARM_SMMU_PRIQ_PROD 0x100c8
174 #define ARM_SMMU_PRIQ_CONS 0x100cc
175 #define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0
176 #define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8
177 #define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
179 /* Common MSI config fields */
180 #define MSI_CFG0_ADDR_SHIFT 2
181 #define MSI_CFG0_ADDR_MASK 0x3fffffffffffUL
182 #define MSI_CFG2_SH_SHIFT 4
183 #define MSI_CFG2_SH_NSH (0UL << MSI_CFG2_SH_SHIFT)
184 #define MSI_CFG2_SH_OSH (2UL << MSI_CFG2_SH_SHIFT)
185 #define MSI_CFG2_SH_ISH (3UL << MSI_CFG2_SH_SHIFT)
186 #define MSI_CFG2_MEMATTR_SHIFT 0
187 #define MSI_CFG2_MEMATTR_DEVICE_nGnRE (0x1 << MSI_CFG2_MEMATTR_SHIFT)
189 #define Q_IDX(q, p) ((p) & ((1 << (q)->max_n_shift) - 1))
190 #define Q_WRP(q, p) ((p) & (1 << (q)->max_n_shift))
191 #define Q_OVERFLOW_FLAG (1 << 31)
192 #define Q_OVF(q, p) ((p) & Q_OVERFLOW_FLAG)
193 #define Q_ENT(q, p) ((q)->base + \
194 Q_IDX(q, p) * (q)->ent_dwords)
196 #define Q_BASE_RWA (1UL << 62)
197 #define Q_BASE_ADDR_SHIFT 5
198 #define Q_BASE_ADDR_MASK 0xfffffffffffUL
199 #define Q_BASE_LOG2SIZE_SHIFT 0
200 #define Q_BASE_LOG2SIZE_MASK 0x1fUL
205 * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
206 * 2lvl: 128k L1 entries,
207 * 256 lazy entries per table (each table covers a PCI bus)
209 #define STRTAB_L1_SZ_SHIFT 20
210 #define STRTAB_SPLIT 8
212 #define STRTAB_L1_DESC_DWORDS 1
213 #define STRTAB_L1_DESC_SPAN_SHIFT 0
214 #define STRTAB_L1_DESC_SPAN_MASK 0x1fUL
215 #define STRTAB_L1_DESC_L2PTR_SHIFT 6
216 #define STRTAB_L1_DESC_L2PTR_MASK 0x3ffffffffffUL
218 #define STRTAB_STE_DWORDS 8
219 #define STRTAB_STE_0_V (1UL << 0)
220 #define STRTAB_STE_0_CFG_SHIFT 1
221 #define STRTAB_STE_0_CFG_MASK 0x7UL
222 #define STRTAB_STE_0_CFG_ABORT (0UL << STRTAB_STE_0_CFG_SHIFT)
223 #define STRTAB_STE_0_CFG_BYPASS (4UL << STRTAB_STE_0_CFG_SHIFT)
224 #define STRTAB_STE_0_CFG_S1_TRANS (5UL << STRTAB_STE_0_CFG_SHIFT)
225 #define STRTAB_STE_0_CFG_S2_TRANS (6UL << STRTAB_STE_0_CFG_SHIFT)
227 #define STRTAB_STE_0_S1FMT_SHIFT 4
228 #define STRTAB_STE_0_S1FMT_LINEAR (0UL << STRTAB_STE_0_S1FMT_SHIFT)
229 #define STRTAB_STE_0_S1CTXPTR_SHIFT 6
230 #define STRTAB_STE_0_S1CTXPTR_MASK 0x3ffffffffffUL
231 #define STRTAB_STE_0_S1CDMAX_SHIFT 59
232 #define STRTAB_STE_0_S1CDMAX_MASK 0x1fUL
234 #define STRTAB_STE_1_S1C_CACHE_NC 0UL
235 #define STRTAB_STE_1_S1C_CACHE_WBRA 1UL
236 #define STRTAB_STE_1_S1C_CACHE_WT 2UL
237 #define STRTAB_STE_1_S1C_CACHE_WB 3UL
238 #define STRTAB_STE_1_S1C_SH_NSH 0UL
239 #define STRTAB_STE_1_S1C_SH_OSH 2UL
240 #define STRTAB_STE_1_S1C_SH_ISH 3UL
241 #define STRTAB_STE_1_S1CIR_SHIFT 2
242 #define STRTAB_STE_1_S1COR_SHIFT 4
243 #define STRTAB_STE_1_S1CSH_SHIFT 6
245 #define STRTAB_STE_1_S1STALLD (1UL << 27)
247 #define STRTAB_STE_1_EATS_ABT 0UL
248 #define STRTAB_STE_1_EATS_TRANS 1UL
249 #define STRTAB_STE_1_EATS_S1CHK 2UL
250 #define STRTAB_STE_1_EATS_SHIFT 28
252 #define STRTAB_STE_1_STRW_NSEL1 0UL
253 #define STRTAB_STE_1_STRW_EL2 2UL
254 #define STRTAB_STE_1_STRW_SHIFT 30
256 #define STRTAB_STE_2_S2VMID_SHIFT 0
257 #define STRTAB_STE_2_S2VMID_MASK 0xffffUL
258 #define STRTAB_STE_2_VTCR_SHIFT 32
259 #define STRTAB_STE_2_VTCR_MASK 0x7ffffUL
260 #define STRTAB_STE_2_S2AA64 (1UL << 51)
261 #define STRTAB_STE_2_S2ENDI (1UL << 52)
262 #define STRTAB_STE_2_S2PTW (1UL << 54)
263 #define STRTAB_STE_2_S2R (1UL << 58)
265 #define STRTAB_STE_3_S2TTB_SHIFT 4
266 #define STRTAB_STE_3_S2TTB_MASK 0xfffffffffffUL
268 /* Context descriptor (stage-1 only) */
269 #define CTXDESC_CD_DWORDS 8
270 #define CTXDESC_CD_0_TCR_T0SZ_SHIFT 0
271 #define ARM64_TCR_T0SZ_SHIFT 0
272 #define ARM64_TCR_T0SZ_MASK 0x1fUL
273 #define CTXDESC_CD_0_TCR_TG0_SHIFT 6
274 #define ARM64_TCR_TG0_SHIFT 14
275 #define ARM64_TCR_TG0_MASK 0x3UL
276 #define CTXDESC_CD_0_TCR_IRGN0_SHIFT 8
277 #define ARM64_TCR_IRGN0_SHIFT 8
278 #define ARM64_TCR_IRGN0_MASK 0x3UL
279 #define CTXDESC_CD_0_TCR_ORGN0_SHIFT 10
280 #define ARM64_TCR_ORGN0_SHIFT 10
281 #define ARM64_TCR_ORGN0_MASK 0x3UL
282 #define CTXDESC_CD_0_TCR_SH0_SHIFT 12
283 #define ARM64_TCR_SH0_SHIFT 12
284 #define ARM64_TCR_SH0_MASK 0x3UL
285 #define CTXDESC_CD_0_TCR_EPD0_SHIFT 14
286 #define ARM64_TCR_EPD0_SHIFT 7
287 #define ARM64_TCR_EPD0_MASK 0x1UL
288 #define CTXDESC_CD_0_TCR_EPD1_SHIFT 30
289 #define ARM64_TCR_EPD1_SHIFT 23
290 #define ARM64_TCR_EPD1_MASK 0x1UL
292 #define CTXDESC_CD_0_ENDI (1UL << 15)
293 #define CTXDESC_CD_0_V (1UL << 31)
295 #define CTXDESC_CD_0_TCR_IPS_SHIFT 32
296 #define ARM64_TCR_IPS_SHIFT 32
297 #define ARM64_TCR_IPS_MASK 0x7UL
298 #define CTXDESC_CD_0_TCR_TBI0_SHIFT 38
299 #define ARM64_TCR_TBI0_SHIFT 37
300 #define ARM64_TCR_TBI0_MASK 0x1UL
302 #define CTXDESC_CD_0_AA64 (1UL << 41)
303 #define CTXDESC_CD_0_R (1UL << 45)
304 #define CTXDESC_CD_0_A (1UL << 46)
305 #define CTXDESC_CD_0_ASET_SHIFT 47
306 #define CTXDESC_CD_0_ASET_SHARED (0UL << CTXDESC_CD_0_ASET_SHIFT)
307 #define CTXDESC_CD_0_ASET_PRIVATE (1UL << CTXDESC_CD_0_ASET_SHIFT)
308 #define CTXDESC_CD_0_ASID_SHIFT 48
309 #define CTXDESC_CD_0_ASID_MASK 0xffffUL
311 #define CTXDESC_CD_1_TTB0_SHIFT 4
312 #define CTXDESC_CD_1_TTB0_MASK 0xfffffffffffUL
314 #define CTXDESC_CD_3_MAIR_SHIFT 0
316 /* Convert between AArch64 (CPU) TCR format and SMMU CD format */
317 #define ARM_SMMU_TCR2CD(tcr, fld) \
318 (((tcr) >> ARM64_TCR_##fld##_SHIFT & ARM64_TCR_##fld##_MASK) \
319 << CTXDESC_CD_0_TCR_##fld##_SHIFT)
322 #define CMDQ_ENT_DWORDS 2
323 #define CMDQ_MAX_SZ_SHIFT 8
325 #define CMDQ_ERR_SHIFT 24
326 #define CMDQ_ERR_MASK 0x7f
327 #define CMDQ_ERR_CERROR_NONE_IDX 0
328 #define CMDQ_ERR_CERROR_ILL_IDX 1
329 #define CMDQ_ERR_CERROR_ABT_IDX 2
331 #define CMDQ_0_OP_SHIFT 0
332 #define CMDQ_0_OP_MASK 0xffUL
333 #define CMDQ_0_SSV (1UL << 11)
335 #define CMDQ_PREFETCH_0_SID_SHIFT 32
336 #define CMDQ_PREFETCH_1_SIZE_SHIFT 0
337 #define CMDQ_PREFETCH_1_ADDR_MASK ~0xfffUL
339 #define CMDQ_CFGI_0_SID_SHIFT 32
340 #define CMDQ_CFGI_0_SID_MASK 0xffffffffUL
341 #define CMDQ_CFGI_1_LEAF (1UL << 0)
342 #define CMDQ_CFGI_1_RANGE_SHIFT 0
343 #define CMDQ_CFGI_1_RANGE_MASK 0x1fUL
345 #define CMDQ_TLBI_0_VMID_SHIFT 32
346 #define CMDQ_TLBI_0_ASID_SHIFT 48
347 #define CMDQ_TLBI_1_LEAF (1UL << 0)
348 #define CMDQ_TLBI_1_VA_MASK ~0xfffUL
349 #define CMDQ_TLBI_1_IPA_MASK 0xfffffffff000UL
351 #define CMDQ_PRI_0_SSID_SHIFT 12
352 #define CMDQ_PRI_0_SSID_MASK 0xfffffUL
353 #define CMDQ_PRI_0_SID_SHIFT 32
354 #define CMDQ_PRI_0_SID_MASK 0xffffffffUL
355 #define CMDQ_PRI_1_GRPID_SHIFT 0
356 #define CMDQ_PRI_1_GRPID_MASK 0x1ffUL
357 #define CMDQ_PRI_1_RESP_SHIFT 12
358 #define CMDQ_PRI_1_RESP_DENY (0UL << CMDQ_PRI_1_RESP_SHIFT)
359 #define CMDQ_PRI_1_RESP_FAIL (1UL << CMDQ_PRI_1_RESP_SHIFT)
360 #define CMDQ_PRI_1_RESP_SUCC (2UL << CMDQ_PRI_1_RESP_SHIFT)
362 #define CMDQ_SYNC_0_CS_SHIFT 12
363 #define CMDQ_SYNC_0_CS_NONE (0UL << CMDQ_SYNC_0_CS_SHIFT)
364 #define CMDQ_SYNC_0_CS_SEV (2UL << CMDQ_SYNC_0_CS_SHIFT)
367 #define EVTQ_ENT_DWORDS 4
368 #define EVTQ_MAX_SZ_SHIFT 7
370 #define EVTQ_0_ID_SHIFT 0
371 #define EVTQ_0_ID_MASK 0xffUL
374 #define PRIQ_ENT_DWORDS 2
375 #define PRIQ_MAX_SZ_SHIFT 8
377 #define PRIQ_0_SID_SHIFT 0
378 #define PRIQ_0_SID_MASK 0xffffffffUL
379 #define PRIQ_0_SSID_SHIFT 32
380 #define PRIQ_0_SSID_MASK 0xfffffUL
381 #define PRIQ_0_OF (1UL << 57)
382 #define PRIQ_0_PERM_PRIV (1UL << 58)
383 #define PRIQ_0_PERM_EXEC (1UL << 59)
384 #define PRIQ_0_PERM_READ (1UL << 60)
385 #define PRIQ_0_PERM_WRITE (1UL << 61)
386 #define PRIQ_0_PRG_LAST (1UL << 62)
387 #define PRIQ_0_SSID_V (1UL << 63)
389 #define PRIQ_1_PRG_IDX_SHIFT 0
390 #define PRIQ_1_PRG_IDX_MASK 0x1ffUL
391 #define PRIQ_1_ADDR_SHIFT 12
392 #define PRIQ_1_ADDR_MASK 0xfffffffffffffUL
394 /* High-level queue structures */
395 #define ARM_SMMU_POLL_TIMEOUT_US 100
397 static bool disable_bypass;
398 module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO);
399 MODULE_PARM_DESC(disable_bypass,
400 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
408 enum arm_smmu_msi_index {
415 static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
417 ARM_SMMU_EVTQ_IRQ_CFG0,
418 ARM_SMMU_EVTQ_IRQ_CFG1,
419 ARM_SMMU_EVTQ_IRQ_CFG2,
421 [GERROR_MSI_INDEX] = {
422 ARM_SMMU_GERROR_IRQ_CFG0,
423 ARM_SMMU_GERROR_IRQ_CFG1,
424 ARM_SMMU_GERROR_IRQ_CFG2,
427 ARM_SMMU_PRIQ_IRQ_CFG0,
428 ARM_SMMU_PRIQ_IRQ_CFG1,
429 ARM_SMMU_PRIQ_IRQ_CFG2,
433 struct arm_smmu_cmdq_ent {
436 bool substream_valid;
438 /* Command-specific fields */
440 #define CMDQ_OP_PREFETCH_CFG 0x1
447 #define CMDQ_OP_CFGI_STE 0x3
448 #define CMDQ_OP_CFGI_ALL 0x4
457 #define CMDQ_OP_TLBI_NH_ASID 0x11
458 #define CMDQ_OP_TLBI_NH_VA 0x12
459 #define CMDQ_OP_TLBI_EL2_ALL 0x20
460 #define CMDQ_OP_TLBI_S12_VMALL 0x28
461 #define CMDQ_OP_TLBI_S2_IPA 0x2a
462 #define CMDQ_OP_TLBI_NSNH_ALL 0x30
470 #define CMDQ_OP_PRI_RESP 0x41
478 #define CMDQ_OP_CMD_SYNC 0x46
482 struct arm_smmu_queue {
483 int irq; /* Wired interrupt */
494 u32 __iomem *prod_reg;
495 u32 __iomem *cons_reg;
498 struct arm_smmu_cmdq {
499 struct arm_smmu_queue q;
503 struct arm_smmu_evtq {
504 struct arm_smmu_queue q;
508 struct arm_smmu_priq {
509 struct arm_smmu_queue q;
512 /* High-level stream table and context descriptor structures */
513 struct arm_smmu_strtab_l1_desc {
517 dma_addr_t l2ptr_dma;
520 struct arm_smmu_s1_cfg {
522 dma_addr_t cdptr_dma;
524 struct arm_smmu_ctx_desc {
532 struct arm_smmu_s2_cfg {
538 struct arm_smmu_strtab_ent {
541 bool bypass; /* Overrides s1/s2 config */
542 struct arm_smmu_s1_cfg *s1_cfg;
543 struct arm_smmu_s2_cfg *s2_cfg;
546 struct arm_smmu_strtab_cfg {
548 dma_addr_t strtab_dma;
549 struct arm_smmu_strtab_l1_desc *l1_desc;
550 unsigned int num_l1_ents;
556 /* An SMMUv3 instance */
557 struct arm_smmu_device {
561 #define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0)
562 #define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1)
563 #define ARM_SMMU_FEAT_TT_LE (1 << 2)
564 #define ARM_SMMU_FEAT_TT_BE (1 << 3)
565 #define ARM_SMMU_FEAT_PRI (1 << 4)
566 #define ARM_SMMU_FEAT_ATS (1 << 5)
567 #define ARM_SMMU_FEAT_SEV (1 << 6)
568 #define ARM_SMMU_FEAT_MSI (1 << 7)
569 #define ARM_SMMU_FEAT_COHERENCY (1 << 8)
570 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 9)
571 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 10)
572 #define ARM_SMMU_FEAT_STALLS (1 << 11)
573 #define ARM_SMMU_FEAT_HYP (1 << 12)
576 #define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0)
579 struct arm_smmu_cmdq cmdq;
580 struct arm_smmu_evtq evtq;
581 struct arm_smmu_priq priq;
585 unsigned long ias; /* IPA */
586 unsigned long oas; /* PA */
588 #define ARM_SMMU_MAX_ASIDS (1 << 16)
589 unsigned int asid_bits;
590 DECLARE_BITMAP(asid_map, ARM_SMMU_MAX_ASIDS);
592 #define ARM_SMMU_MAX_VMIDS (1 << 16)
593 unsigned int vmid_bits;
594 DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
596 unsigned int ssid_bits;
597 unsigned int sid_bits;
599 struct arm_smmu_strtab_cfg strtab_cfg;
602 /* SMMU private data for an IOMMU group */
603 struct arm_smmu_group {
604 struct arm_smmu_device *smmu;
605 struct arm_smmu_domain *domain;
608 struct arm_smmu_strtab_ent ste;
611 /* SMMU private data for an IOMMU domain */
612 enum arm_smmu_domain_stage {
613 ARM_SMMU_DOMAIN_S1 = 0,
615 ARM_SMMU_DOMAIN_NESTED,
618 struct arm_smmu_domain {
619 struct arm_smmu_device *smmu;
620 struct mutex init_mutex; /* Protects smmu pointer */
622 struct io_pgtable_ops *pgtbl_ops;
623 spinlock_t pgtbl_lock;
625 enum arm_smmu_domain_stage stage;
627 struct arm_smmu_s1_cfg s1_cfg;
628 struct arm_smmu_s2_cfg s2_cfg;
631 struct iommu_domain domain;
634 struct arm_smmu_option_prop {
639 static struct arm_smmu_option_prop arm_smmu_options[] = {
640 { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
644 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
646 return container_of(dom, struct arm_smmu_domain, domain);
649 static void parse_driver_options(struct arm_smmu_device *smmu)
654 if (of_property_read_bool(smmu->dev->of_node,
655 arm_smmu_options[i].prop)) {
656 smmu->options |= arm_smmu_options[i].opt;
657 dev_notice(smmu->dev, "option %s\n",
658 arm_smmu_options[i].prop);
660 } while (arm_smmu_options[++i].opt);
663 /* Low-level queue manipulation functions */
664 static bool queue_full(struct arm_smmu_queue *q)
666 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
667 Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
670 static bool queue_empty(struct arm_smmu_queue *q)
672 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
673 Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
676 static void queue_sync_cons(struct arm_smmu_queue *q)
678 q->cons = readl_relaxed(q->cons_reg);
681 static void queue_inc_cons(struct arm_smmu_queue *q)
683 u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
685 q->cons = Q_OVF(q, q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
688 * Ensure that all CPU accesses (reads and writes) to the queue
689 * are complete before we update the cons pointer.
692 writel_relaxed(q->cons, q->cons_reg);
695 static int queue_sync_prod(struct arm_smmu_queue *q)
698 u32 prod = readl_relaxed(q->prod_reg);
700 if (Q_OVF(q, prod) != Q_OVF(q, q->prod))
707 static void queue_inc_prod(struct arm_smmu_queue *q)
709 u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + 1;
711 q->prod = Q_OVF(q, q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
712 writel(q->prod, q->prod_reg);
715 static bool __queue_cons_before(struct arm_smmu_queue *q, u32 until)
717 if (Q_WRP(q, q->cons) == Q_WRP(q, until))
718 return Q_IDX(q, q->cons) < Q_IDX(q, until);
720 return Q_IDX(q, q->cons) >= Q_IDX(q, until);
723 static int queue_poll_cons(struct arm_smmu_queue *q, u32 until, bool wfe)
725 ktime_t timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
727 while (queue_sync_cons(q), __queue_cons_before(q, until)) {
728 if (ktime_compare(ktime_get(), timeout) > 0)
742 static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
746 for (i = 0; i < n_dwords; ++i)
747 *dst++ = cpu_to_le64(*src++);
750 static int queue_insert_raw(struct arm_smmu_queue *q, u64 *ent)
755 queue_write(Q_ENT(q, q->prod), ent, q->ent_dwords);
760 static void queue_read(__le64 *dst, u64 *src, size_t n_dwords)
764 for (i = 0; i < n_dwords; ++i)
765 *dst++ = le64_to_cpu(*src++);
768 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
773 queue_read(ent, Q_ENT(q, q->cons), q->ent_dwords);
778 /* High-level queue accessors */
779 static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
781 memset(cmd, 0, CMDQ_ENT_DWORDS << 3);
782 cmd[0] |= (ent->opcode & CMDQ_0_OP_MASK) << CMDQ_0_OP_SHIFT;
784 switch (ent->opcode) {
785 case CMDQ_OP_TLBI_EL2_ALL:
786 case CMDQ_OP_TLBI_NSNH_ALL:
788 case CMDQ_OP_PREFETCH_CFG:
789 cmd[0] |= (u64)ent->prefetch.sid << CMDQ_PREFETCH_0_SID_SHIFT;
790 cmd[1] |= ent->prefetch.size << CMDQ_PREFETCH_1_SIZE_SHIFT;
791 cmd[1] |= ent->prefetch.addr & CMDQ_PREFETCH_1_ADDR_MASK;
793 case CMDQ_OP_CFGI_STE:
794 cmd[0] |= (u64)ent->cfgi.sid << CMDQ_CFGI_0_SID_SHIFT;
795 cmd[1] |= ent->cfgi.leaf ? CMDQ_CFGI_1_LEAF : 0;
797 case CMDQ_OP_CFGI_ALL:
798 /* Cover the entire SID range */
799 cmd[1] |= CMDQ_CFGI_1_RANGE_MASK << CMDQ_CFGI_1_RANGE_SHIFT;
801 case CMDQ_OP_TLBI_NH_VA:
802 cmd[0] |= (u64)ent->tlbi.asid << CMDQ_TLBI_0_ASID_SHIFT;
803 cmd[1] |= ent->tlbi.leaf ? CMDQ_TLBI_1_LEAF : 0;
804 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
806 case CMDQ_OP_TLBI_S2_IPA:
807 cmd[0] |= (u64)ent->tlbi.vmid << CMDQ_TLBI_0_VMID_SHIFT;
808 cmd[1] |= ent->tlbi.leaf ? CMDQ_TLBI_1_LEAF : 0;
809 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
811 case CMDQ_OP_TLBI_NH_ASID:
812 cmd[0] |= (u64)ent->tlbi.asid << CMDQ_TLBI_0_ASID_SHIFT;
814 case CMDQ_OP_TLBI_S12_VMALL:
815 cmd[0] |= (u64)ent->tlbi.vmid << CMDQ_TLBI_0_VMID_SHIFT;
817 case CMDQ_OP_PRI_RESP:
818 cmd[0] |= ent->substream_valid ? CMDQ_0_SSV : 0;
819 cmd[0] |= ent->pri.ssid << CMDQ_PRI_0_SSID_SHIFT;
820 cmd[0] |= (u64)ent->pri.sid << CMDQ_PRI_0_SID_SHIFT;
821 cmd[1] |= ent->pri.grpid << CMDQ_PRI_1_GRPID_SHIFT;
822 switch (ent->pri.resp) {
824 cmd[1] |= CMDQ_PRI_1_RESP_DENY;
827 cmd[1] |= CMDQ_PRI_1_RESP_FAIL;
830 cmd[1] |= CMDQ_PRI_1_RESP_SUCC;
836 case CMDQ_OP_CMD_SYNC:
837 cmd[0] |= CMDQ_SYNC_0_CS_SEV;
846 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
848 static const char *cerror_str[] = {
849 [CMDQ_ERR_CERROR_NONE_IDX] = "No error",
850 [CMDQ_ERR_CERROR_ILL_IDX] = "Illegal command",
851 [CMDQ_ERR_CERROR_ABT_IDX] = "Abort on command fetch",
855 u64 cmd[CMDQ_ENT_DWORDS];
856 struct arm_smmu_queue *q = &smmu->cmdq.q;
857 u32 cons = readl_relaxed(q->cons_reg);
858 u32 idx = cons >> CMDQ_ERR_SHIFT & CMDQ_ERR_MASK;
859 struct arm_smmu_cmdq_ent cmd_sync = {
860 .opcode = CMDQ_OP_CMD_SYNC,
863 dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
867 case CMDQ_ERR_CERROR_ILL_IDX:
869 case CMDQ_ERR_CERROR_ABT_IDX:
870 dev_err(smmu->dev, "retrying command fetch\n");
871 case CMDQ_ERR_CERROR_NONE_IDX:
876 * We may have concurrent producers, so we need to be careful
877 * not to touch any of the shadow cmdq state.
879 queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
880 dev_err(smmu->dev, "skipping command in error state:\n");
881 for (i = 0; i < ARRAY_SIZE(cmd); ++i)
882 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
884 /* Convert the erroneous command into a CMD_SYNC */
885 if (arm_smmu_cmdq_build_cmd(cmd, &cmd_sync)) {
886 dev_err(smmu->dev, "failed to convert to CMD_SYNC\n");
890 queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
893 static void arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
894 struct arm_smmu_cmdq_ent *ent)
897 u64 cmd[CMDQ_ENT_DWORDS];
898 bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
899 struct arm_smmu_queue *q = &smmu->cmdq.q;
901 if (arm_smmu_cmdq_build_cmd(cmd, ent)) {
902 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
907 spin_lock(&smmu->cmdq.lock);
908 while (until = q->prod + 1, queue_insert_raw(q, cmd) == -ENOSPC) {
910 * Keep the queue locked, otherwise the producer could wrap
911 * twice and we could see a future consumer pointer that looks
912 * like it's behind us.
914 if (queue_poll_cons(q, until, wfe))
915 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
918 if (ent->opcode == CMDQ_OP_CMD_SYNC && queue_poll_cons(q, until, wfe))
919 dev_err_ratelimited(smmu->dev, "CMD_SYNC timeout\n");
920 spin_unlock(&smmu->cmdq.lock);
923 /* Context descriptor manipulation functions */
924 static u64 arm_smmu_cpu_tcr_to_cd(u64 tcr)
928 /* Repack the TCR. Just care about TTBR0 for now */
929 val |= ARM_SMMU_TCR2CD(tcr, T0SZ);
930 val |= ARM_SMMU_TCR2CD(tcr, TG0);
931 val |= ARM_SMMU_TCR2CD(tcr, IRGN0);
932 val |= ARM_SMMU_TCR2CD(tcr, ORGN0);
933 val |= ARM_SMMU_TCR2CD(tcr, SH0);
934 val |= ARM_SMMU_TCR2CD(tcr, EPD0);
935 val |= ARM_SMMU_TCR2CD(tcr, EPD1);
936 val |= ARM_SMMU_TCR2CD(tcr, IPS);
937 val |= ARM_SMMU_TCR2CD(tcr, TBI0);
942 static void arm_smmu_write_ctx_desc(struct arm_smmu_device *smmu,
943 struct arm_smmu_s1_cfg *cfg)
948 * We don't need to issue any invalidation here, as we'll invalidate
949 * the STE when installing the new entry anyway.
951 val = arm_smmu_cpu_tcr_to_cd(cfg->cd.tcr) |
955 CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET_PRIVATE |
956 CTXDESC_CD_0_AA64 | (u64)cfg->cd.asid << CTXDESC_CD_0_ASID_SHIFT |
958 cfg->cdptr[0] = cpu_to_le64(val);
960 val = cfg->cd.ttbr & CTXDESC_CD_1_TTB0_MASK << CTXDESC_CD_1_TTB0_SHIFT;
961 cfg->cdptr[1] = cpu_to_le64(val);
963 cfg->cdptr[3] = cpu_to_le64(cfg->cd.mair << CTXDESC_CD_3_MAIR_SHIFT);
966 /* Stream table manipulation functions */
968 arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc)
972 val |= (desc->span & STRTAB_L1_DESC_SPAN_MASK)
973 << STRTAB_L1_DESC_SPAN_SHIFT;
974 val |= desc->l2ptr_dma &
975 STRTAB_L1_DESC_L2PTR_MASK << STRTAB_L1_DESC_L2PTR_SHIFT;
977 *dst = cpu_to_le64(val);
980 static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid)
982 struct arm_smmu_cmdq_ent cmd = {
983 .opcode = CMDQ_OP_CFGI_STE,
990 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
991 cmd.opcode = CMDQ_OP_CMD_SYNC;
992 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
995 static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid,
996 __le64 *dst, struct arm_smmu_strtab_ent *ste)
999 * This is hideously complicated, but we only really care about
1000 * three cases at the moment:
1002 * 1. Invalid (all zero) -> bypass (init)
1003 * 2. Bypass -> translation (attach)
1004 * 3. Translation -> bypass (detach)
1006 * Given that we can't update the STE atomically and the SMMU
1007 * doesn't read the thing in a defined order, that leaves us
1008 * with the following maintenance requirements:
1010 * 1. Update Config, return (init time STEs aren't live)
1011 * 2. Write everything apart from dword 0, sync, write dword 0, sync
1012 * 3. Update Config, sync
1014 u64 val = le64_to_cpu(dst[0]);
1015 bool ste_live = false;
1016 struct arm_smmu_cmdq_ent prefetch_cmd = {
1017 .opcode = CMDQ_OP_PREFETCH_CFG,
1023 if (val & STRTAB_STE_0_V) {
1026 cfg = val & STRTAB_STE_0_CFG_MASK << STRTAB_STE_0_CFG_SHIFT;
1028 case STRTAB_STE_0_CFG_BYPASS:
1030 case STRTAB_STE_0_CFG_S1_TRANS:
1031 case STRTAB_STE_0_CFG_S2_TRANS:
1034 case STRTAB_STE_0_CFG_ABORT:
1038 BUG(); /* STE corruption */
1042 /* Nuke the existing STE_0 value, as we're going to rewrite it */
1043 val = ste->valid ? STRTAB_STE_0_V : 0;
1046 val |= disable_bypass ? STRTAB_STE_0_CFG_ABORT
1047 : STRTAB_STE_0_CFG_BYPASS;
1048 dst[0] = cpu_to_le64(val);
1049 dst[2] = 0; /* Nuke the VMID */
1051 arm_smmu_sync_ste_for_sid(smmu, sid);
1057 dst[1] = cpu_to_le64(
1058 STRTAB_STE_1_S1C_CACHE_WBRA
1059 << STRTAB_STE_1_S1CIR_SHIFT |
1060 STRTAB_STE_1_S1C_CACHE_WBRA
1061 << STRTAB_STE_1_S1COR_SHIFT |
1062 STRTAB_STE_1_S1C_SH_ISH << STRTAB_STE_1_S1CSH_SHIFT |
1063 STRTAB_STE_1_S1STALLD |
1064 #ifdef CONFIG_PCI_ATS
1065 STRTAB_STE_1_EATS_TRANS << STRTAB_STE_1_EATS_SHIFT |
1067 STRTAB_STE_1_STRW_NSEL1 << STRTAB_STE_1_STRW_SHIFT);
1069 val |= (ste->s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK
1070 << STRTAB_STE_0_S1CTXPTR_SHIFT) |
1071 STRTAB_STE_0_CFG_S1_TRANS;
1076 dst[2] = cpu_to_le64(
1077 ste->s2_cfg->vmid << STRTAB_STE_2_S2VMID_SHIFT |
1078 (ste->s2_cfg->vtcr & STRTAB_STE_2_VTCR_MASK)
1079 << STRTAB_STE_2_VTCR_SHIFT |
1081 STRTAB_STE_2_S2ENDI |
1083 STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 |
1086 dst[3] = cpu_to_le64(ste->s2_cfg->vttbr &
1087 STRTAB_STE_3_S2TTB_MASK << STRTAB_STE_3_S2TTB_SHIFT);
1089 val |= STRTAB_STE_0_CFG_S2_TRANS;
1092 arm_smmu_sync_ste_for_sid(smmu, sid);
1093 /* See comment in arm_smmu_write_ctx_desc() */
1094 WRITE_ONCE(dst[0], cpu_to_le64(val));
1095 arm_smmu_sync_ste_for_sid(smmu, sid);
1097 /* It's likely that we'll want to use the new STE soon */
1098 if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH))
1099 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
1102 static void arm_smmu_init_bypass_stes(u64 *strtab, unsigned int nent)
1105 struct arm_smmu_strtab_ent ste = {
1110 for (i = 0; i < nent; ++i) {
1111 arm_smmu_write_strtab_ent(NULL, -1, strtab, &ste);
1112 strtab += STRTAB_STE_DWORDS;
1116 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
1120 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1121 struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT];
1126 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
1127 strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS];
1129 desc->span = STRTAB_SPLIT + 1;
1130 desc->l2ptr = dma_zalloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
1134 "failed to allocate l2 stream table for SID %u\n",
1139 arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT);
1140 arm_smmu_write_strtab_l1_desc(strtab, desc);
1144 /* IRQ and event handlers */
1145 static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
1148 struct arm_smmu_device *smmu = dev;
1149 struct arm_smmu_queue *q = &smmu->evtq.q;
1150 u64 evt[EVTQ_ENT_DWORDS];
1152 while (!queue_remove_raw(q, evt)) {
1153 u8 id = evt[0] >> EVTQ_0_ID_SHIFT & EVTQ_0_ID_MASK;
1155 dev_info(smmu->dev, "event 0x%02x received:\n", id);
1156 for (i = 0; i < ARRAY_SIZE(evt); ++i)
1157 dev_info(smmu->dev, "\t0x%016llx\n",
1158 (unsigned long long)evt[i]);
1161 /* Sync our overflow flag, as we believe we're up to speed */
1162 q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
1166 static irqreturn_t arm_smmu_evtq_handler(int irq, void *dev)
1168 irqreturn_t ret = IRQ_WAKE_THREAD;
1169 struct arm_smmu_device *smmu = dev;
1170 struct arm_smmu_queue *q = &smmu->evtq.q;
1173 * Not much we can do on overflow, so scream and pretend we're
1176 if (queue_sync_prod(q) == -EOVERFLOW)
1177 dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
1178 else if (queue_empty(q))
1184 static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
1186 struct arm_smmu_device *smmu = dev;
1187 struct arm_smmu_queue *q = &smmu->priq.q;
1188 u64 evt[PRIQ_ENT_DWORDS];
1190 while (!queue_remove_raw(q, evt)) {
1195 sid = evt[0] >> PRIQ_0_SID_SHIFT & PRIQ_0_SID_MASK;
1196 ssv = evt[0] & PRIQ_0_SSID_V;
1197 ssid = ssv ? evt[0] >> PRIQ_0_SSID_SHIFT & PRIQ_0_SSID_MASK : 0;
1198 last = evt[0] & PRIQ_0_PRG_LAST;
1199 grpid = evt[1] >> PRIQ_1_PRG_IDX_SHIFT & PRIQ_1_PRG_IDX_MASK;
1201 dev_info(smmu->dev, "unexpected PRI request received:\n");
1203 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
1204 sid, ssid, grpid, last ? "L" : "",
1205 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
1206 evt[0] & PRIQ_0_PERM_READ ? "R" : "",
1207 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
1208 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
1209 evt[1] & PRIQ_1_ADDR_MASK << PRIQ_1_ADDR_SHIFT);
1212 struct arm_smmu_cmdq_ent cmd = {
1213 .opcode = CMDQ_OP_PRI_RESP,
1214 .substream_valid = ssv,
1219 .resp = PRI_RESP_DENY,
1223 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1227 /* Sync our overflow flag, as we believe we're up to speed */
1228 q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
1229 writel(q->cons, q->cons_reg);
1233 static irqreturn_t arm_smmu_priq_handler(int irq, void *dev)
1235 irqreturn_t ret = IRQ_WAKE_THREAD;
1236 struct arm_smmu_device *smmu = dev;
1237 struct arm_smmu_queue *q = &smmu->priq.q;
1239 /* PRIQ overflow indicates a programming error */
1240 if (queue_sync_prod(q) == -EOVERFLOW)
1241 dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
1242 else if (queue_empty(q))
1248 static irqreturn_t arm_smmu_cmdq_sync_handler(int irq, void *dev)
1250 /* We don't actually use CMD_SYNC interrupts for anything */
1254 static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
1256 static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
1258 u32 gerror, gerrorn;
1259 struct arm_smmu_device *smmu = dev;
1261 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
1262 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
1265 if (!(gerror & GERROR_ERR_MASK))
1266 return IRQ_NONE; /* No errors pending */
1269 "unexpected global error reported (0x%08x), this could be serious\n",
1272 if (gerror & GERROR_SFM_ERR) {
1273 dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
1274 arm_smmu_device_disable(smmu);
1277 if (gerror & GERROR_MSI_GERROR_ABT_ERR)
1278 dev_warn(smmu->dev, "GERROR MSI write aborted\n");
1280 if (gerror & GERROR_MSI_PRIQ_ABT_ERR) {
1281 dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
1282 arm_smmu_priq_handler(irq, smmu->dev);
1285 if (gerror & GERROR_MSI_EVTQ_ABT_ERR) {
1286 dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
1287 arm_smmu_evtq_handler(irq, smmu->dev);
1290 if (gerror & GERROR_MSI_CMDQ_ABT_ERR) {
1291 dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
1292 arm_smmu_cmdq_sync_handler(irq, smmu->dev);
1295 if (gerror & GERROR_PRIQ_ABT_ERR)
1296 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
1298 if (gerror & GERROR_EVTQ_ABT_ERR)
1299 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
1301 if (gerror & GERROR_CMDQ_ERR)
1302 arm_smmu_cmdq_skip_err(smmu);
1304 writel(gerror, smmu->base + ARM_SMMU_GERRORN);
1308 /* IO_PGTABLE API */
1309 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
1311 struct arm_smmu_cmdq_ent cmd;
1313 cmd.opcode = CMDQ_OP_CMD_SYNC;
1314 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1317 static void arm_smmu_tlb_sync(void *cookie)
1319 struct arm_smmu_domain *smmu_domain = cookie;
1320 __arm_smmu_tlb_sync(smmu_domain->smmu);
1323 static void arm_smmu_tlb_inv_context(void *cookie)
1325 struct arm_smmu_domain *smmu_domain = cookie;
1326 struct arm_smmu_device *smmu = smmu_domain->smmu;
1327 struct arm_smmu_cmdq_ent cmd;
1329 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1330 cmd.opcode = CMDQ_OP_TLBI_NH_ASID;
1331 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
1334 cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
1335 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
1338 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1339 __arm_smmu_tlb_sync(smmu);
1342 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
1343 bool leaf, void *cookie)
1345 struct arm_smmu_domain *smmu_domain = cookie;
1346 struct arm_smmu_device *smmu = smmu_domain->smmu;
1347 struct arm_smmu_cmdq_ent cmd = {
1354 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1355 cmd.opcode = CMDQ_OP_TLBI_NH_VA;
1356 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
1358 cmd.opcode = CMDQ_OP_TLBI_S2_IPA;
1359 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
1362 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1365 static struct iommu_gather_ops arm_smmu_gather_ops = {
1366 .tlb_flush_all = arm_smmu_tlb_inv_context,
1367 .tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
1368 .tlb_sync = arm_smmu_tlb_sync,
1372 static bool arm_smmu_capable(enum iommu_cap cap)
1375 case IOMMU_CAP_CACHE_COHERENCY:
1377 case IOMMU_CAP_INTR_REMAP:
1378 return true; /* MSIs are just memory writes */
1379 case IOMMU_CAP_NOEXEC:
1386 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1388 struct arm_smmu_domain *smmu_domain;
1390 if (type != IOMMU_DOMAIN_UNMANAGED)
1394 * Allocate the domain and initialise some of its data structures.
1395 * We can't really do anything meaningful until we've added a
1398 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1402 mutex_init(&smmu_domain->init_mutex);
1403 spin_lock_init(&smmu_domain->pgtbl_lock);
1404 return &smmu_domain->domain;
1407 static int arm_smmu_bitmap_alloc(unsigned long *map, int span)
1409 int idx, size = 1 << span;
1412 idx = find_first_zero_bit(map, size);
1415 } while (test_and_set_bit(idx, map));
1420 static void arm_smmu_bitmap_free(unsigned long *map, int idx)
1422 clear_bit(idx, map);
1425 static void arm_smmu_domain_free(struct iommu_domain *domain)
1427 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1428 struct arm_smmu_device *smmu = smmu_domain->smmu;
1430 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1432 /* Free the CD and ASID, if we allocated them */
1433 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1434 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1437 dma_free_coherent(smmu_domain->smmu->dev,
1438 CTXDESC_CD_DWORDS << 3,
1442 arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid);
1445 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
1447 arm_smmu_bitmap_free(smmu->vmid_map, cfg->vmid);
1453 static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
1454 struct io_pgtable_cfg *pgtbl_cfg)
1458 struct arm_smmu_device *smmu = smmu_domain->smmu;
1459 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1461 asid = arm_smmu_bitmap_alloc(smmu->asid_map, smmu->asid_bits);
1462 if (IS_ERR_VALUE(asid))
1465 cfg->cdptr = dma_zalloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3,
1466 &cfg->cdptr_dma, GFP_KERNEL);
1468 dev_warn(smmu->dev, "failed to allocate context descriptor\n");
1473 cfg->cd.asid = (u16)asid;
1474 cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
1475 cfg->cd.tcr = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
1476 cfg->cd.mair = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
1480 arm_smmu_bitmap_free(smmu->asid_map, asid);
1484 static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,
1485 struct io_pgtable_cfg *pgtbl_cfg)
1488 struct arm_smmu_device *smmu = smmu_domain->smmu;
1489 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
1491 vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits);
1492 if (IS_ERR_VALUE(vmid))
1495 cfg->vmid = (u16)vmid;
1496 cfg->vttbr = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
1497 cfg->vtcr = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
1501 static struct iommu_ops arm_smmu_ops;
1503 static int arm_smmu_domain_finalise(struct iommu_domain *domain)
1506 unsigned long ias, oas;
1507 enum io_pgtable_fmt fmt;
1508 struct io_pgtable_cfg pgtbl_cfg;
1509 struct io_pgtable_ops *pgtbl_ops;
1510 int (*finalise_stage_fn)(struct arm_smmu_domain *,
1511 struct io_pgtable_cfg *);
1512 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1513 struct arm_smmu_device *smmu = smmu_domain->smmu;
1515 /* Restrict the stage to what we can actually support */
1516 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
1517 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
1518 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
1519 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1521 switch (smmu_domain->stage) {
1522 case ARM_SMMU_DOMAIN_S1:
1525 fmt = ARM_64_LPAE_S1;
1526 finalise_stage_fn = arm_smmu_domain_finalise_s1;
1528 case ARM_SMMU_DOMAIN_NESTED:
1529 case ARM_SMMU_DOMAIN_S2:
1532 fmt = ARM_64_LPAE_S2;
1533 finalise_stage_fn = arm_smmu_domain_finalise_s2;
1539 pgtbl_cfg = (struct io_pgtable_cfg) {
1540 .pgsize_bitmap = arm_smmu_ops.pgsize_bitmap,
1543 .tlb = &arm_smmu_gather_ops,
1544 .iommu_dev = smmu->dev,
1547 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
1551 arm_smmu_ops.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
1553 ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg);
1554 if (IS_ERR_VALUE(ret)) {
1555 free_io_pgtable_ops(pgtbl_ops);
1559 smmu_domain->pgtbl_ops = pgtbl_ops;
1563 static struct arm_smmu_group *arm_smmu_group_get(struct device *dev)
1565 struct iommu_group *group;
1566 struct arm_smmu_group *smmu_group;
1568 group = iommu_group_get(dev);
1572 smmu_group = iommu_group_get_iommudata(group);
1573 iommu_group_put(group);
1577 static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
1580 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1582 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
1583 struct arm_smmu_strtab_l1_desc *l1_desc;
1586 /* Two-level walk */
1587 idx = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS;
1588 l1_desc = &cfg->l1_desc[idx];
1589 idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
1590 step = &l1_desc->l2ptr[idx];
1592 /* Simple linear lookup */
1593 step = &cfg->strtab[sid * STRTAB_STE_DWORDS];
1599 static int arm_smmu_install_ste_for_group(struct arm_smmu_group *smmu_group)
1602 struct arm_smmu_domain *smmu_domain = smmu_group->domain;
1603 struct arm_smmu_strtab_ent *ste = &smmu_group->ste;
1604 struct arm_smmu_device *smmu = smmu_group->smmu;
1606 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1607 ste->s1_cfg = &smmu_domain->s1_cfg;
1609 arm_smmu_write_ctx_desc(smmu, ste->s1_cfg);
1612 ste->s2_cfg = &smmu_domain->s2_cfg;
1615 for (i = 0; i < smmu_group->num_sids; ++i) {
1616 u32 sid = smmu_group->sids[i];
1617 __le64 *step = arm_smmu_get_step_for_sid(smmu, sid);
1619 arm_smmu_write_strtab_ent(smmu, sid, step, ste);
1625 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1628 struct arm_smmu_device *smmu;
1629 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1630 struct arm_smmu_group *smmu_group = arm_smmu_group_get(dev);
1635 /* Already attached to a different domain? */
1636 if (smmu_group->domain && smmu_group->domain != smmu_domain)
1639 smmu = smmu_group->smmu;
1640 mutex_lock(&smmu_domain->init_mutex);
1642 if (!smmu_domain->smmu) {
1643 smmu_domain->smmu = smmu;
1644 ret = arm_smmu_domain_finalise(domain);
1646 smmu_domain->smmu = NULL;
1649 } else if (smmu_domain->smmu != smmu) {
1651 "cannot attach to SMMU %s (upstream of %s)\n",
1652 dev_name(smmu_domain->smmu->dev),
1653 dev_name(smmu->dev));
1658 /* Group already attached to this domain? */
1659 if (smmu_group->domain)
1662 smmu_group->domain = smmu_domain;
1663 smmu_group->ste.bypass = false;
1665 ret = arm_smmu_install_ste_for_group(smmu_group);
1666 if (IS_ERR_VALUE(ret))
1667 smmu_group->domain = NULL;
1670 mutex_unlock(&smmu_domain->init_mutex);
1674 static void arm_smmu_detach_dev(struct iommu_domain *domain, struct device *dev)
1676 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1677 struct arm_smmu_group *smmu_group = arm_smmu_group_get(dev);
1679 BUG_ON(!smmu_domain);
1680 BUG_ON(!smmu_group);
1682 mutex_lock(&smmu_domain->init_mutex);
1683 BUG_ON(smmu_group->domain != smmu_domain);
1685 smmu_group->ste.bypass = true;
1686 if (IS_ERR_VALUE(arm_smmu_install_ste_for_group(smmu_group)))
1687 dev_warn(dev, "failed to install bypass STE\n");
1689 smmu_group->domain = NULL;
1690 mutex_unlock(&smmu_domain->init_mutex);
1693 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1694 phys_addr_t paddr, size_t size, int prot)
1697 unsigned long flags;
1698 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1699 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1704 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1705 ret = ops->map(ops, iova, paddr, size, prot);
1706 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1711 arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
1714 unsigned long flags;
1715 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1716 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1721 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1722 ret = ops->unmap(ops, iova, size);
1723 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1728 arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
1731 unsigned long flags;
1732 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1733 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1738 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1739 ret = ops->iova_to_phys(ops, iova);
1740 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1745 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *sidp)
1747 *(u32 *)sidp = alias;
1748 return 0; /* Continue walking */
1751 static void __arm_smmu_release_pci_iommudata(void *data)
1756 static struct arm_smmu_device *arm_smmu_get_for_pci_dev(struct pci_dev *pdev)
1758 struct device_node *of_node;
1759 struct platform_device *smmu_pdev;
1760 struct arm_smmu_device *smmu = NULL;
1761 struct pci_bus *bus = pdev->bus;
1763 /* Walk up to the root bus */
1764 while (!pci_is_root_bus(bus))
1767 /* Follow the "iommus" phandle from the host controller */
1768 of_node = of_parse_phandle(bus->bridge->parent->of_node, "iommus", 0);
1772 /* See if we can find an SMMU corresponding to the phandle */
1773 smmu_pdev = of_find_device_by_node(of_node);
1775 smmu = platform_get_drvdata(smmu_pdev);
1777 of_node_put(of_node);
1781 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
1783 unsigned long limit = smmu->strtab_cfg.num_l1_ents;
1785 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
1786 limit *= 1UL << STRTAB_SPLIT;
1791 static int arm_smmu_add_device(struct device *dev)
1795 struct pci_dev *pdev;
1796 struct iommu_group *group;
1797 struct arm_smmu_group *smmu_group;
1798 struct arm_smmu_device *smmu;
1800 /* We only support PCI, for now */
1801 if (!dev_is_pci(dev))
1804 pdev = to_pci_dev(dev);
1805 group = iommu_group_get_for_dev(dev);
1807 return PTR_ERR(group);
1809 smmu_group = iommu_group_get_iommudata(group);
1811 smmu = arm_smmu_get_for_pci_dev(pdev);
1817 smmu_group = kzalloc(sizeof(*smmu_group), GFP_KERNEL);
1823 smmu_group->ste.valid = true;
1824 smmu_group->smmu = smmu;
1825 iommu_group_set_iommudata(group, smmu_group,
1826 __arm_smmu_release_pci_iommudata);
1828 smmu = smmu_group->smmu;
1831 /* Assume SID == RID until firmware tells us otherwise */
1832 pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid, &sid);
1833 for (i = 0; i < smmu_group->num_sids; ++i) {
1834 /* If we already know about this SID, then we're done */
1835 if (smmu_group->sids[i] == sid)
1839 /* Check the SID is in range of the SMMU and our stream table */
1840 if (!arm_smmu_sid_in_range(smmu, sid)) {
1845 /* Ensure l2 strtab is initialised */
1846 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
1847 ret = arm_smmu_init_l2_strtab(smmu, sid);
1852 /* Resize the SID array for the group */
1853 smmu_group->num_sids++;
1854 sids = krealloc(smmu_group->sids, smmu_group->num_sids * sizeof(*sids),
1857 smmu_group->num_sids--;
1862 /* Add the new SID */
1863 sids[smmu_group->num_sids - 1] = sid;
1864 smmu_group->sids = sids;
1868 iommu_group_put(group);
1872 static void arm_smmu_remove_device(struct device *dev)
1874 iommu_group_remove_device(dev);
1877 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1878 enum iommu_attr attr, void *data)
1880 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1883 case DOMAIN_ATTR_NESTING:
1884 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1891 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1892 enum iommu_attr attr, void *data)
1895 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1897 mutex_lock(&smmu_domain->init_mutex);
1900 case DOMAIN_ATTR_NESTING:
1901 if (smmu_domain->smmu) {
1907 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1909 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1917 mutex_unlock(&smmu_domain->init_mutex);
1921 static struct iommu_ops arm_smmu_ops = {
1922 .capable = arm_smmu_capable,
1923 .domain_alloc = arm_smmu_domain_alloc,
1924 .domain_free = arm_smmu_domain_free,
1925 .attach_dev = arm_smmu_attach_dev,
1926 .detach_dev = arm_smmu_detach_dev,
1927 .map = arm_smmu_map,
1928 .unmap = arm_smmu_unmap,
1929 .map_sg = default_iommu_map_sg,
1930 .iova_to_phys = arm_smmu_iova_to_phys,
1931 .add_device = arm_smmu_add_device,
1932 .remove_device = arm_smmu_remove_device,
1933 .device_group = pci_device_group,
1934 .domain_get_attr = arm_smmu_domain_get_attr,
1935 .domain_set_attr = arm_smmu_domain_set_attr,
1936 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1939 /* Probing and initialisation functions */
1940 static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
1941 struct arm_smmu_queue *q,
1942 unsigned long prod_off,
1943 unsigned long cons_off,
1946 size_t qsz = ((1 << q->max_n_shift) * dwords) << 3;
1948 q->base = dma_alloc_coherent(smmu->dev, qsz, &q->base_dma, GFP_KERNEL);
1950 dev_err(smmu->dev, "failed to allocate queue (0x%zx bytes)\n",
1955 q->prod_reg = smmu->base + prod_off;
1956 q->cons_reg = smmu->base + cons_off;
1957 q->ent_dwords = dwords;
1959 q->q_base = Q_BASE_RWA;
1960 q->q_base |= q->base_dma & Q_BASE_ADDR_MASK << Q_BASE_ADDR_SHIFT;
1961 q->q_base |= (q->max_n_shift & Q_BASE_LOG2SIZE_MASK)
1962 << Q_BASE_LOG2SIZE_SHIFT;
1964 q->prod = q->cons = 0;
1968 static void arm_smmu_free_one_queue(struct arm_smmu_device *smmu,
1969 struct arm_smmu_queue *q)
1971 size_t qsz = ((1 << q->max_n_shift) * q->ent_dwords) << 3;
1973 dma_free_coherent(smmu->dev, qsz, q->base, q->base_dma);
1976 static void arm_smmu_free_queues(struct arm_smmu_device *smmu)
1978 arm_smmu_free_one_queue(smmu, &smmu->cmdq.q);
1979 arm_smmu_free_one_queue(smmu, &smmu->evtq.q);
1981 if (smmu->features & ARM_SMMU_FEAT_PRI)
1982 arm_smmu_free_one_queue(smmu, &smmu->priq.q);
1985 static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
1990 spin_lock_init(&smmu->cmdq.lock);
1991 ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD,
1992 ARM_SMMU_CMDQ_CONS, CMDQ_ENT_DWORDS);
1997 ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD,
1998 ARM_SMMU_EVTQ_CONS, EVTQ_ENT_DWORDS);
2003 if (!(smmu->features & ARM_SMMU_FEAT_PRI))
2006 ret = arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD,
2007 ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS);
2014 arm_smmu_free_one_queue(smmu, &smmu->evtq.q);
2016 arm_smmu_free_one_queue(smmu, &smmu->cmdq.q);
2021 static void arm_smmu_free_l2_strtab(struct arm_smmu_device *smmu)
2025 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2027 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
2028 for (i = 0; i < cfg->num_l1_ents; ++i) {
2029 struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[i];
2034 dma_free_coherent(smmu->dev, size, desc->l2ptr,
2039 static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu)
2042 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2043 size_t size = sizeof(*cfg->l1_desc) * cfg->num_l1_ents;
2044 void *strtab = smmu->strtab_cfg.strtab;
2046 cfg->l1_desc = devm_kzalloc(smmu->dev, size, GFP_KERNEL);
2047 if (!cfg->l1_desc) {
2048 dev_err(smmu->dev, "failed to allocate l1 stream table desc\n");
2052 for (i = 0; i < cfg->num_l1_ents; ++i) {
2053 arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]);
2054 strtab += STRTAB_L1_DESC_DWORDS << 3;
2060 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
2066 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2069 * If we can resolve everything with a single L2 table, then we
2070 * just need a single L1 descriptor. Otherwise, calculate the L1
2071 * size, capped to the SIDSIZE.
2073 if (smmu->sid_bits < STRTAB_SPLIT) {
2076 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
2077 size = min(size, smmu->sid_bits - STRTAB_SPLIT);
2079 cfg->num_l1_ents = 1 << size;
2081 size += STRTAB_SPLIT;
2082 if (size < smmu->sid_bits)
2084 "2-level strtab only covers %u/%u bits of SID\n",
2085 size, smmu->sid_bits);
2087 l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);
2088 strtab = dma_zalloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
2092 "failed to allocate l1 stream table (%u bytes)\n",
2096 cfg->strtab = strtab;
2098 /* Configure strtab_base_cfg for 2 levels */
2099 reg = STRTAB_BASE_CFG_FMT_2LVL;
2100 reg |= (size & STRTAB_BASE_CFG_LOG2SIZE_MASK)
2101 << STRTAB_BASE_CFG_LOG2SIZE_SHIFT;
2102 reg |= (STRTAB_SPLIT & STRTAB_BASE_CFG_SPLIT_MASK)
2103 << STRTAB_BASE_CFG_SPLIT_SHIFT;
2104 cfg->strtab_base_cfg = reg;
2106 ret = arm_smmu_init_l1_strtab(smmu);
2108 dma_free_coherent(smmu->dev,
2115 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
2120 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2122 size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);
2123 strtab = dma_zalloc_coherent(smmu->dev, size, &cfg->strtab_dma,
2127 "failed to allocate linear stream table (%u bytes)\n",
2131 cfg->strtab = strtab;
2132 cfg->num_l1_ents = 1 << smmu->sid_bits;
2134 /* Configure strtab_base_cfg for a linear table covering all SIDs */
2135 reg = STRTAB_BASE_CFG_FMT_LINEAR;
2136 reg |= (smmu->sid_bits & STRTAB_BASE_CFG_LOG2SIZE_MASK)
2137 << STRTAB_BASE_CFG_LOG2SIZE_SHIFT;
2138 cfg->strtab_base_cfg = reg;
2140 arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents);
2144 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
2149 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2150 ret = arm_smmu_init_strtab_2lvl(smmu);
2152 ret = arm_smmu_init_strtab_linear(smmu);
2157 /* Set the strtab base address */
2158 reg = smmu->strtab_cfg.strtab_dma &
2159 STRTAB_BASE_ADDR_MASK << STRTAB_BASE_ADDR_SHIFT;
2160 reg |= STRTAB_BASE_RA;
2161 smmu->strtab_cfg.strtab_base = reg;
2163 /* Allocate the first VMID for stage-2 bypass STEs */
2164 set_bit(0, smmu->vmid_map);
2168 static void arm_smmu_free_strtab(struct arm_smmu_device *smmu)
2170 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2171 u32 size = cfg->num_l1_ents;
2173 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2174 arm_smmu_free_l2_strtab(smmu);
2175 size *= STRTAB_L1_DESC_DWORDS << 3;
2177 size *= STRTAB_STE_DWORDS * 3;
2180 dma_free_coherent(smmu->dev, size, cfg->strtab, cfg->strtab_dma);
2183 static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
2187 ret = arm_smmu_init_queues(smmu);
2191 ret = arm_smmu_init_strtab(smmu);
2193 goto out_free_queues;
2198 arm_smmu_free_queues(smmu);
2202 static void arm_smmu_free_structures(struct arm_smmu_device *smmu)
2204 arm_smmu_free_strtab(smmu);
2205 arm_smmu_free_queues(smmu);
2208 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
2209 unsigned int reg_off, unsigned int ack_off)
2213 writel_relaxed(val, smmu->base + reg_off);
2214 return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
2215 1, ARM_SMMU_POLL_TIMEOUT_US);
2218 static void arm_smmu_free_msis(void *data)
2220 struct device *dev = data;
2221 platform_msi_domain_free_irqs(dev);
2224 static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
2226 phys_addr_t doorbell;
2227 struct device *dev = msi_desc_to_dev(desc);
2228 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2229 phys_addr_t *cfg = arm_smmu_msi_cfg[desc->platform.msi_index];
2231 doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;
2232 doorbell &= MSI_CFG0_ADDR_MASK << MSI_CFG0_ADDR_SHIFT;
2234 writeq_relaxed(doorbell, smmu->base + cfg[0]);
2235 writel_relaxed(msg->data, smmu->base + cfg[1]);
2236 writel_relaxed(MSI_CFG2_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);
2239 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu)
2241 struct msi_desc *desc;
2242 int ret, nvec = ARM_SMMU_MAX_MSIS;
2243 struct device *dev = smmu->dev;
2245 /* Clear the MSI address regs */
2246 writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
2247 writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
2249 if (smmu->features & ARM_SMMU_FEAT_PRI)
2250 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
2254 if (!(smmu->features & ARM_SMMU_FEAT_MSI))
2257 /* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */
2258 ret = platform_msi_domain_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg);
2260 dev_warn(dev, "failed to allocate MSIs\n");
2264 for_each_msi_entry(desc, dev) {
2265 switch (desc->platform.msi_index) {
2266 case EVTQ_MSI_INDEX:
2267 smmu->evtq.q.irq = desc->irq;
2269 case GERROR_MSI_INDEX:
2270 smmu->gerr_irq = desc->irq;
2272 case PRIQ_MSI_INDEX:
2273 smmu->priq.q.irq = desc->irq;
2275 default: /* Unknown */
2280 /* Add callback to free MSIs on teardown */
2281 devm_add_action(dev, arm_smmu_free_msis, dev);
2284 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
2287 u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
2289 /* Disable IRQs first */
2290 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
2291 ARM_SMMU_IRQ_CTRLACK);
2293 dev_err(smmu->dev, "failed to disable irqs\n");
2297 arm_smmu_setup_msis(smmu);
2299 /* Request interrupt lines */
2300 irq = smmu->evtq.q.irq;
2302 ret = devm_request_threaded_irq(smmu->dev, irq,
2303 arm_smmu_evtq_handler,
2304 arm_smmu_evtq_thread,
2305 0, "arm-smmu-v3-evtq", smmu);
2306 if (IS_ERR_VALUE(ret))
2307 dev_warn(smmu->dev, "failed to enable evtq irq\n");
2310 irq = smmu->cmdq.q.irq;
2312 ret = devm_request_irq(smmu->dev, irq,
2313 arm_smmu_cmdq_sync_handler, 0,
2314 "arm-smmu-v3-cmdq-sync", smmu);
2315 if (IS_ERR_VALUE(ret))
2316 dev_warn(smmu->dev, "failed to enable cmdq-sync irq\n");
2319 irq = smmu->gerr_irq;
2321 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
2322 0, "arm-smmu-v3-gerror", smmu);
2323 if (IS_ERR_VALUE(ret))
2324 dev_warn(smmu->dev, "failed to enable gerror irq\n");
2327 if (smmu->features & ARM_SMMU_FEAT_PRI) {
2328 irq = smmu->priq.q.irq;
2330 ret = devm_request_threaded_irq(smmu->dev, irq,
2331 arm_smmu_priq_handler,
2332 arm_smmu_priq_thread,
2333 0, "arm-smmu-v3-priq",
2335 if (IS_ERR_VALUE(ret))
2337 "failed to enable priq irq\n");
2339 irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
2343 /* Enable interrupt generation on the SMMU */
2344 ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
2345 ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
2347 dev_warn(smmu->dev, "failed to enable irqs\n");
2352 static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
2356 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
2358 dev_err(smmu->dev, "failed to clear cr0\n");
2363 static int arm_smmu_device_reset(struct arm_smmu_device *smmu)
2367 struct arm_smmu_cmdq_ent cmd;
2369 /* Clear CR0 and sync (disables SMMU and queue processing) */
2370 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
2371 if (reg & CR0_SMMUEN)
2372 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
2374 ret = arm_smmu_device_disable(smmu);
2378 /* CR1 (table and queue memory attributes) */
2379 reg = (CR1_SH_ISH << CR1_TABLE_SH_SHIFT) |
2380 (CR1_CACHE_WB << CR1_TABLE_OC_SHIFT) |
2381 (CR1_CACHE_WB << CR1_TABLE_IC_SHIFT) |
2382 (CR1_SH_ISH << CR1_QUEUE_SH_SHIFT) |
2383 (CR1_CACHE_WB << CR1_QUEUE_OC_SHIFT) |
2384 (CR1_CACHE_WB << CR1_QUEUE_IC_SHIFT);
2385 writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
2387 /* CR2 (random crap) */
2388 reg = CR2_PTM | CR2_RECINVSID | CR2_E2H;
2389 writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
2392 writeq_relaxed(smmu->strtab_cfg.strtab_base,
2393 smmu->base + ARM_SMMU_STRTAB_BASE);
2394 writel_relaxed(smmu->strtab_cfg.strtab_base_cfg,
2395 smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
2398 writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
2399 writel_relaxed(smmu->cmdq.q.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
2400 writel_relaxed(smmu->cmdq.q.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
2402 enables = CR0_CMDQEN;
2403 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2406 dev_err(smmu->dev, "failed to enable command queue\n");
2410 /* Invalidate any cached configuration */
2411 cmd.opcode = CMDQ_OP_CFGI_ALL;
2412 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2413 cmd.opcode = CMDQ_OP_CMD_SYNC;
2414 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2416 /* Invalidate any stale TLB entries */
2417 if (smmu->features & ARM_SMMU_FEAT_HYP) {
2418 cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
2419 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2422 cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
2423 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2424 cmd.opcode = CMDQ_OP_CMD_SYNC;
2425 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2428 writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
2429 writel_relaxed(smmu->evtq.q.prod, smmu->base + ARM_SMMU_EVTQ_PROD);
2430 writel_relaxed(smmu->evtq.q.cons, smmu->base + ARM_SMMU_EVTQ_CONS);
2432 enables |= CR0_EVTQEN;
2433 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2436 dev_err(smmu->dev, "failed to enable event queue\n");
2441 if (smmu->features & ARM_SMMU_FEAT_PRI) {
2442 writeq_relaxed(smmu->priq.q.q_base,
2443 smmu->base + ARM_SMMU_PRIQ_BASE);
2444 writel_relaxed(smmu->priq.q.prod,
2445 smmu->base + ARM_SMMU_PRIQ_PROD);
2446 writel_relaxed(smmu->priq.q.cons,
2447 smmu->base + ARM_SMMU_PRIQ_CONS);
2449 enables |= CR0_PRIQEN;
2450 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2453 dev_err(smmu->dev, "failed to enable PRI queue\n");
2458 ret = arm_smmu_setup_irqs(smmu);
2460 dev_err(smmu->dev, "failed to setup irqs\n");
2464 /* Enable the SMMU interface */
2465 enables |= CR0_SMMUEN;
2466 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2469 dev_err(smmu->dev, "failed to enable SMMU interface\n");
2476 static int arm_smmu_device_probe(struct arm_smmu_device *smmu)
2480 unsigned long pgsize_bitmap = 0;
2483 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
2485 /* 2-level structures */
2486 if ((reg & IDR0_ST_LVL_MASK << IDR0_ST_LVL_SHIFT) == IDR0_ST_LVL_2LVL)
2487 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
2489 if (reg & IDR0_CD2L)
2490 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
2493 * Translation table endianness.
2494 * We currently require the same endianness as the CPU, but this
2495 * could be changed later by adding a new IO_PGTABLE_QUIRK.
2497 switch (reg & IDR0_TTENDIAN_MASK << IDR0_TTENDIAN_SHIFT) {
2498 case IDR0_TTENDIAN_MIXED:
2499 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
2502 case IDR0_TTENDIAN_BE:
2503 smmu->features |= ARM_SMMU_FEAT_TT_BE;
2506 case IDR0_TTENDIAN_LE:
2507 smmu->features |= ARM_SMMU_FEAT_TT_LE;
2511 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
2515 /* Boolean feature flags */
2516 if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
2517 smmu->features |= ARM_SMMU_FEAT_PRI;
2519 if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
2520 smmu->features |= ARM_SMMU_FEAT_ATS;
2523 smmu->features |= ARM_SMMU_FEAT_SEV;
2526 smmu->features |= ARM_SMMU_FEAT_MSI;
2529 smmu->features |= ARM_SMMU_FEAT_HYP;
2532 * The dma-coherent property is used in preference to the ID
2533 * register, but warn on mismatch.
2535 coherent = of_dma_is_coherent(smmu->dev->of_node);
2537 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
2539 if (!!(reg & IDR0_COHACC) != coherent)
2540 dev_warn(smmu->dev, "IDR0.COHACC overridden by dma-coherent property (%s)\n",
2541 coherent ? "true" : "false");
2543 if (reg & IDR0_STALL_MODEL)
2544 smmu->features |= ARM_SMMU_FEAT_STALLS;
2547 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
2550 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
2552 if (!(reg & (IDR0_S1P | IDR0_S2P))) {
2553 dev_err(smmu->dev, "no translation support!\n");
2557 /* We only support the AArch64 table format at present */
2558 switch (reg & IDR0_TTF_MASK << IDR0_TTF_SHIFT) {
2559 case IDR0_TTF_AARCH32_64:
2562 case IDR0_TTF_AARCH64:
2565 dev_err(smmu->dev, "AArch64 table format not supported!\n");
2569 /* ASID/VMID sizes */
2570 smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
2571 smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
2574 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
2575 if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
2576 dev_err(smmu->dev, "embedded implementation not supported\n");
2580 /* Queue sizes, capped at 4k */
2581 smmu->cmdq.q.max_n_shift = min((u32)CMDQ_MAX_SZ_SHIFT,
2582 reg >> IDR1_CMDQ_SHIFT & IDR1_CMDQ_MASK);
2583 if (!smmu->cmdq.q.max_n_shift) {
2584 /* Odd alignment restrictions on the base, so ignore for now */
2585 dev_err(smmu->dev, "unit-length command queue not supported\n");
2589 smmu->evtq.q.max_n_shift = min((u32)EVTQ_MAX_SZ_SHIFT,
2590 reg >> IDR1_EVTQ_SHIFT & IDR1_EVTQ_MASK);
2591 smmu->priq.q.max_n_shift = min((u32)PRIQ_MAX_SZ_SHIFT,
2592 reg >> IDR1_PRIQ_SHIFT & IDR1_PRIQ_MASK);
2594 /* SID/SSID sizes */
2595 smmu->ssid_bits = reg >> IDR1_SSID_SHIFT & IDR1_SSID_MASK;
2596 smmu->sid_bits = reg >> IDR1_SID_SHIFT & IDR1_SID_MASK;
2599 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
2601 /* Maximum number of outstanding stalls */
2602 smmu->evtq.max_stalls = reg >> IDR5_STALL_MAX_SHIFT
2603 & IDR5_STALL_MAX_MASK;
2606 if (reg & IDR5_GRAN64K)
2607 pgsize_bitmap |= SZ_64K | SZ_512M;
2608 if (reg & IDR5_GRAN16K)
2609 pgsize_bitmap |= SZ_16K | SZ_32M;
2610 if (reg & IDR5_GRAN4K)
2611 pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
2613 arm_smmu_ops.pgsize_bitmap &= pgsize_bitmap;
2615 /* Output address size */
2616 switch (reg & IDR5_OAS_MASK << IDR5_OAS_SHIFT) {
2617 case IDR5_OAS_32_BIT:
2620 case IDR5_OAS_36_BIT:
2623 case IDR5_OAS_40_BIT:
2626 case IDR5_OAS_42_BIT:
2629 case IDR5_OAS_44_BIT:
2634 "unknown output address size. Truncating to 48-bit\n");
2636 case IDR5_OAS_48_BIT:
2640 /* Set the DMA mask for our table walker */
2641 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
2643 "failed to set DMA mask for table walker\n");
2645 smmu->ias = max(smmu->ias, smmu->oas);
2647 dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
2648 smmu->ias, smmu->oas, smmu->features);
2652 static int arm_smmu_device_dt_probe(struct platform_device *pdev)
2655 struct resource *res;
2656 struct arm_smmu_device *smmu;
2657 struct device *dev = &pdev->dev;
2659 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2661 dev_err(dev, "failed to allocate arm_smmu_device\n");
2667 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2668 if (resource_size(res) + 1 < SZ_128K) {
2669 dev_err(dev, "MMIO region too small (%pr)\n", res);
2673 smmu->base = devm_ioremap_resource(dev, res);
2674 if (IS_ERR(smmu->base))
2675 return PTR_ERR(smmu->base);
2677 /* Interrupt lines */
2678 irq = platform_get_irq_byname(pdev, "eventq");
2680 smmu->evtq.q.irq = irq;
2682 irq = platform_get_irq_byname(pdev, "priq");
2684 smmu->priq.q.irq = irq;
2686 irq = platform_get_irq_byname(pdev, "cmdq-sync");
2688 smmu->cmdq.q.irq = irq;
2690 irq = platform_get_irq_byname(pdev, "gerror");
2692 smmu->gerr_irq = irq;
2694 parse_driver_options(smmu);
2697 ret = arm_smmu_device_probe(smmu);
2701 /* Initialise in-memory data structures */
2702 ret = arm_smmu_init_structures(smmu);
2706 /* Record our private device structure */
2707 platform_set_drvdata(pdev, smmu);
2709 /* Reset the device */
2710 ret = arm_smmu_device_reset(smmu);
2712 goto out_free_structures;
2716 out_free_structures:
2717 arm_smmu_free_structures(smmu);
2721 static int arm_smmu_device_remove(struct platform_device *pdev)
2723 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2725 arm_smmu_device_disable(smmu);
2726 arm_smmu_free_structures(smmu);
2730 static struct of_device_id arm_smmu_of_match[] = {
2731 { .compatible = "arm,smmu-v3", },
2734 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
2736 static struct platform_driver arm_smmu_driver = {
2738 .name = "arm-smmu-v3",
2739 .of_match_table = of_match_ptr(arm_smmu_of_match),
2741 .probe = arm_smmu_device_dt_probe,
2742 .remove = arm_smmu_device_remove,
2745 static int __init arm_smmu_init(void)
2747 struct device_node *np;
2750 np = of_find_matching_node(NULL, arm_smmu_of_match);
2756 ret = platform_driver_register(&arm_smmu_driver);
2760 return bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2763 static void __exit arm_smmu_exit(void)
2765 return platform_driver_unregister(&arm_smmu_driver);
2768 subsys_initcall(arm_smmu_init);
2769 module_exit(arm_smmu_exit);
2771 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
2772 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2773 MODULE_LICENSE("GPL v2");