1 // SPDX-License-Identifier: GPL-2.0
3 * IOMMU API for ARM architected SMMUv3 implementations.
5 * Copyright (C) 2015 ARM Limited
7 * Author: Will Deacon <will.deacon@arm.com>
9 * This driver is powered by bad coffee and bombay mix.
12 #include <linux/acpi.h>
13 #include <linux/acpi_iort.h>
14 #include <linux/bitfield.h>
15 #include <linux/bitops.h>
16 #include <linux/crash_dump.h>
17 #include <linux/delay.h>
18 #include <linux/dma-iommu.h>
19 #include <linux/err.h>
20 #include <linux/interrupt.h>
21 #include <linux/io-pgtable.h>
22 #include <linux/iommu.h>
23 #include <linux/iopoll.h>
24 #include <linux/init.h>
25 #include <linux/moduleparam.h>
26 #include <linux/msi.h>
28 #include <linux/of_address.h>
29 #include <linux/of_iommu.h>
30 #include <linux/of_platform.h>
31 #include <linux/pci.h>
32 #include <linux/pci-ats.h>
33 #include <linux/platform_device.h>
35 #include <linux/amba/bus.h>
38 #define ARM_SMMU_IDR0 0x0
39 #define IDR0_ST_LVL GENMASK(28, 27)
40 #define IDR0_ST_LVL_2LVL 1
41 #define IDR0_STALL_MODEL GENMASK(25, 24)
42 #define IDR0_STALL_MODEL_STALL 0
43 #define IDR0_STALL_MODEL_FORCE 2
44 #define IDR0_TTENDIAN GENMASK(22, 21)
45 #define IDR0_TTENDIAN_MIXED 0
46 #define IDR0_TTENDIAN_LE 2
47 #define IDR0_TTENDIAN_BE 3
48 #define IDR0_CD2L (1 << 19)
49 #define IDR0_VMID16 (1 << 18)
50 #define IDR0_PRI (1 << 16)
51 #define IDR0_SEV (1 << 14)
52 #define IDR0_MSI (1 << 13)
53 #define IDR0_ASID16 (1 << 12)
54 #define IDR0_ATS (1 << 10)
55 #define IDR0_HYP (1 << 9)
56 #define IDR0_COHACC (1 << 4)
57 #define IDR0_TTF GENMASK(3, 2)
58 #define IDR0_TTF_AARCH64 2
59 #define IDR0_TTF_AARCH32_64 3
60 #define IDR0_S1P (1 << 1)
61 #define IDR0_S2P (1 << 0)
63 #define ARM_SMMU_IDR1 0x4
64 #define IDR1_TABLES_PRESET (1 << 30)
65 #define IDR1_QUEUES_PRESET (1 << 29)
66 #define IDR1_REL (1 << 28)
67 #define IDR1_CMDQS GENMASK(25, 21)
68 #define IDR1_EVTQS GENMASK(20, 16)
69 #define IDR1_PRIQS GENMASK(15, 11)
70 #define IDR1_SSIDSIZE GENMASK(10, 6)
71 #define IDR1_SIDSIZE GENMASK(5, 0)
73 #define ARM_SMMU_IDR5 0x14
74 #define IDR5_STALL_MAX GENMASK(31, 16)
75 #define IDR5_GRAN64K (1 << 6)
76 #define IDR5_GRAN16K (1 << 5)
77 #define IDR5_GRAN4K (1 << 4)
78 #define IDR5_OAS GENMASK(2, 0)
79 #define IDR5_OAS_32_BIT 0
80 #define IDR5_OAS_36_BIT 1
81 #define IDR5_OAS_40_BIT 2
82 #define IDR5_OAS_42_BIT 3
83 #define IDR5_OAS_44_BIT 4
84 #define IDR5_OAS_48_BIT 5
85 #define IDR5_OAS_52_BIT 6
86 #define IDR5_VAX GENMASK(11, 10)
87 #define IDR5_VAX_52_BIT 1
89 #define ARM_SMMU_CR0 0x20
90 #define CR0_ATSCHK (1 << 4)
91 #define CR0_CMDQEN (1 << 3)
92 #define CR0_EVTQEN (1 << 2)
93 #define CR0_PRIQEN (1 << 1)
94 #define CR0_SMMUEN (1 << 0)
96 #define ARM_SMMU_CR0ACK 0x24
98 #define ARM_SMMU_CR1 0x28
99 #define CR1_TABLE_SH GENMASK(11, 10)
100 #define CR1_TABLE_OC GENMASK(9, 8)
101 #define CR1_TABLE_IC GENMASK(7, 6)
102 #define CR1_QUEUE_SH GENMASK(5, 4)
103 #define CR1_QUEUE_OC GENMASK(3, 2)
104 #define CR1_QUEUE_IC GENMASK(1, 0)
105 /* CR1 cacheability fields don't quite follow the usual TCR-style encoding */
106 #define CR1_CACHE_NC 0
107 #define CR1_CACHE_WB 1
108 #define CR1_CACHE_WT 2
110 #define ARM_SMMU_CR2 0x2c
111 #define CR2_PTM (1 << 2)
112 #define CR2_RECINVSID (1 << 1)
113 #define CR2_E2H (1 << 0)
115 #define ARM_SMMU_GBPA 0x44
116 #define GBPA_UPDATE (1 << 31)
117 #define GBPA_ABORT (1 << 20)
119 #define ARM_SMMU_IRQ_CTRL 0x50
120 #define IRQ_CTRL_EVTQ_IRQEN (1 << 2)
121 #define IRQ_CTRL_PRIQ_IRQEN (1 << 1)
122 #define IRQ_CTRL_GERROR_IRQEN (1 << 0)
124 #define ARM_SMMU_IRQ_CTRLACK 0x54
126 #define ARM_SMMU_GERROR 0x60
127 #define GERROR_SFM_ERR (1 << 8)
128 #define GERROR_MSI_GERROR_ABT_ERR (1 << 7)
129 #define GERROR_MSI_PRIQ_ABT_ERR (1 << 6)
130 #define GERROR_MSI_EVTQ_ABT_ERR (1 << 5)
131 #define GERROR_MSI_CMDQ_ABT_ERR (1 << 4)
132 #define GERROR_PRIQ_ABT_ERR (1 << 3)
133 #define GERROR_EVTQ_ABT_ERR (1 << 2)
134 #define GERROR_CMDQ_ERR (1 << 0)
135 #define GERROR_ERR_MASK 0xfd
137 #define ARM_SMMU_GERRORN 0x64
139 #define ARM_SMMU_GERROR_IRQ_CFG0 0x68
140 #define ARM_SMMU_GERROR_IRQ_CFG1 0x70
141 #define ARM_SMMU_GERROR_IRQ_CFG2 0x74
143 #define ARM_SMMU_STRTAB_BASE 0x80
144 #define STRTAB_BASE_RA (1UL << 62)
145 #define STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6)
147 #define ARM_SMMU_STRTAB_BASE_CFG 0x88
148 #define STRTAB_BASE_CFG_FMT GENMASK(17, 16)
149 #define STRTAB_BASE_CFG_FMT_LINEAR 0
150 #define STRTAB_BASE_CFG_FMT_2LVL 1
151 #define STRTAB_BASE_CFG_SPLIT GENMASK(10, 6)
152 #define STRTAB_BASE_CFG_LOG2SIZE GENMASK(5, 0)
154 #define ARM_SMMU_CMDQ_BASE 0x90
155 #define ARM_SMMU_CMDQ_PROD 0x98
156 #define ARM_SMMU_CMDQ_CONS 0x9c
158 #define ARM_SMMU_EVTQ_BASE 0xa0
159 #define ARM_SMMU_EVTQ_PROD 0x100a8
160 #define ARM_SMMU_EVTQ_CONS 0x100ac
161 #define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0
162 #define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8
163 #define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc
165 #define ARM_SMMU_PRIQ_BASE 0xc0
166 #define ARM_SMMU_PRIQ_PROD 0x100c8
167 #define ARM_SMMU_PRIQ_CONS 0x100cc
168 #define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0
169 #define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8
170 #define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
172 /* Common MSI config fields */
173 #define MSI_CFG0_ADDR_MASK GENMASK_ULL(51, 2)
174 #define MSI_CFG2_SH GENMASK(5, 4)
175 #define MSI_CFG2_MEMATTR GENMASK(3, 0)
177 /* Common memory attribute values */
178 #define ARM_SMMU_SH_NSH 0
179 #define ARM_SMMU_SH_OSH 2
180 #define ARM_SMMU_SH_ISH 3
181 #define ARM_SMMU_MEMATTR_DEVICE_nGnRE 0x1
182 #define ARM_SMMU_MEMATTR_OIWB 0xf
184 #define Q_IDX(llq, p) ((p) & ((1 << (llq)->max_n_shift) - 1))
185 #define Q_WRP(llq, p) ((p) & (1 << (llq)->max_n_shift))
186 #define Q_OVERFLOW_FLAG (1U << 31)
187 #define Q_OVF(p) ((p) & Q_OVERFLOW_FLAG)
188 #define Q_ENT(q, p) ((q)->base + \
189 Q_IDX(&((q)->llq), p) * \
192 #define Q_BASE_RWA (1UL << 62)
193 #define Q_BASE_ADDR_MASK GENMASK_ULL(51, 5)
194 #define Q_BASE_LOG2SIZE GENMASK(4, 0)
196 /* Ensure DMA allocations are naturally aligned */
197 #ifdef CONFIG_CMA_ALIGNMENT
198 #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + CONFIG_CMA_ALIGNMENT)
200 #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + MAX_ORDER - 1)
206 * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
207 * 2lvl: 128k L1 entries,
208 * 256 lazy entries per table (each table covers a PCI bus)
210 #define STRTAB_L1_SZ_SHIFT 20
211 #define STRTAB_SPLIT 8
213 #define STRTAB_L1_DESC_DWORDS 1
214 #define STRTAB_L1_DESC_SPAN GENMASK_ULL(4, 0)
215 #define STRTAB_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 6)
217 #define STRTAB_STE_DWORDS 8
218 #define STRTAB_STE_0_V (1UL << 0)
219 #define STRTAB_STE_0_CFG GENMASK_ULL(3, 1)
220 #define STRTAB_STE_0_CFG_ABORT 0
221 #define STRTAB_STE_0_CFG_BYPASS 4
222 #define STRTAB_STE_0_CFG_S1_TRANS 5
223 #define STRTAB_STE_0_CFG_S2_TRANS 6
225 #define STRTAB_STE_0_S1FMT GENMASK_ULL(5, 4)
226 #define STRTAB_STE_0_S1FMT_LINEAR 0
227 #define STRTAB_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6)
228 #define STRTAB_STE_0_S1CDMAX GENMASK_ULL(63, 59)
230 #define STRTAB_STE_1_S1C_CACHE_NC 0UL
231 #define STRTAB_STE_1_S1C_CACHE_WBRA 1UL
232 #define STRTAB_STE_1_S1C_CACHE_WT 2UL
233 #define STRTAB_STE_1_S1C_CACHE_WB 3UL
234 #define STRTAB_STE_1_S1CIR GENMASK_ULL(3, 2)
235 #define STRTAB_STE_1_S1COR GENMASK_ULL(5, 4)
236 #define STRTAB_STE_1_S1CSH GENMASK_ULL(7, 6)
238 #define STRTAB_STE_1_S1STALLD (1UL << 27)
240 #define STRTAB_STE_1_EATS GENMASK_ULL(29, 28)
241 #define STRTAB_STE_1_EATS_ABT 0UL
242 #define STRTAB_STE_1_EATS_TRANS 1UL
243 #define STRTAB_STE_1_EATS_S1CHK 2UL
245 #define STRTAB_STE_1_STRW GENMASK_ULL(31, 30)
246 #define STRTAB_STE_1_STRW_NSEL1 0UL
247 #define STRTAB_STE_1_STRW_EL2 2UL
249 #define STRTAB_STE_1_SHCFG GENMASK_ULL(45, 44)
250 #define STRTAB_STE_1_SHCFG_INCOMING 1UL
252 #define STRTAB_STE_2_S2VMID GENMASK_ULL(15, 0)
253 #define STRTAB_STE_2_VTCR GENMASK_ULL(50, 32)
254 #define STRTAB_STE_2_S2AA64 (1UL << 51)
255 #define STRTAB_STE_2_S2ENDI (1UL << 52)
256 #define STRTAB_STE_2_S2PTW (1UL << 54)
257 #define STRTAB_STE_2_S2R (1UL << 58)
259 #define STRTAB_STE_3_S2TTB_MASK GENMASK_ULL(51, 4)
261 /* Context descriptor (stage-1 only) */
262 #define CTXDESC_CD_DWORDS 8
263 #define CTXDESC_CD_0_TCR_T0SZ GENMASK_ULL(5, 0)
264 #define ARM64_TCR_T0SZ GENMASK_ULL(5, 0)
265 #define CTXDESC_CD_0_TCR_TG0 GENMASK_ULL(7, 6)
266 #define ARM64_TCR_TG0 GENMASK_ULL(15, 14)
267 #define CTXDESC_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8)
268 #define ARM64_TCR_IRGN0 GENMASK_ULL(9, 8)
269 #define CTXDESC_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10)
270 #define ARM64_TCR_ORGN0 GENMASK_ULL(11, 10)
271 #define CTXDESC_CD_0_TCR_SH0 GENMASK_ULL(13, 12)
272 #define ARM64_TCR_SH0 GENMASK_ULL(13, 12)
273 #define CTXDESC_CD_0_TCR_EPD0 (1ULL << 14)
274 #define ARM64_TCR_EPD0 (1ULL << 7)
275 #define CTXDESC_CD_0_TCR_EPD1 (1ULL << 30)
276 #define ARM64_TCR_EPD1 (1ULL << 23)
278 #define CTXDESC_CD_0_ENDI (1UL << 15)
279 #define CTXDESC_CD_0_V (1UL << 31)
281 #define CTXDESC_CD_0_TCR_IPS GENMASK_ULL(34, 32)
282 #define ARM64_TCR_IPS GENMASK_ULL(34, 32)
283 #define CTXDESC_CD_0_TCR_TBI0 (1ULL << 38)
284 #define ARM64_TCR_TBI0 (1ULL << 37)
286 #define CTXDESC_CD_0_AA64 (1UL << 41)
287 #define CTXDESC_CD_0_S (1UL << 44)
288 #define CTXDESC_CD_0_R (1UL << 45)
289 #define CTXDESC_CD_0_A (1UL << 46)
290 #define CTXDESC_CD_0_ASET (1UL << 47)
291 #define CTXDESC_CD_0_ASID GENMASK_ULL(63, 48)
293 #define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(51, 4)
295 /* Convert between AArch64 (CPU) TCR format and SMMU CD format */
296 #define ARM_SMMU_TCR2CD(tcr, fld) FIELD_PREP(CTXDESC_CD_0_TCR_##fld, \
297 FIELD_GET(ARM64_TCR_##fld, tcr))
300 #define CMDQ_ENT_SZ_SHIFT 4
301 #define CMDQ_ENT_DWORDS ((1 << CMDQ_ENT_SZ_SHIFT) >> 3)
302 #define CMDQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT)
304 #define CMDQ_CONS_ERR GENMASK(30, 24)
305 #define CMDQ_ERR_CERROR_NONE_IDX 0
306 #define CMDQ_ERR_CERROR_ILL_IDX 1
307 #define CMDQ_ERR_CERROR_ABT_IDX 2
308 #define CMDQ_ERR_CERROR_ATC_INV_IDX 3
310 #define CMDQ_PROD_OWNED_FLAG Q_OVERFLOW_FLAG
313 * This is used to size the command queue and therefore must be at least
314 * BITS_PER_LONG so that the valid_map works correctly (it relies on the
315 * total number of queue entries being a multiple of BITS_PER_LONG).
317 #define CMDQ_BATCH_ENTRIES BITS_PER_LONG
319 #define CMDQ_0_OP GENMASK_ULL(7, 0)
320 #define CMDQ_0_SSV (1UL << 11)
322 #define CMDQ_PREFETCH_0_SID GENMASK_ULL(63, 32)
323 #define CMDQ_PREFETCH_1_SIZE GENMASK_ULL(4, 0)
324 #define CMDQ_PREFETCH_1_ADDR_MASK GENMASK_ULL(63, 12)
326 #define CMDQ_CFGI_0_SID GENMASK_ULL(63, 32)
327 #define CMDQ_CFGI_1_LEAF (1UL << 0)
328 #define CMDQ_CFGI_1_RANGE GENMASK_ULL(4, 0)
330 #define CMDQ_TLBI_0_VMID GENMASK_ULL(47, 32)
331 #define CMDQ_TLBI_0_ASID GENMASK_ULL(63, 48)
332 #define CMDQ_TLBI_1_LEAF (1UL << 0)
333 #define CMDQ_TLBI_1_VA_MASK GENMASK_ULL(63, 12)
334 #define CMDQ_TLBI_1_IPA_MASK GENMASK_ULL(51, 12)
336 #define CMDQ_ATC_0_SSID GENMASK_ULL(31, 12)
337 #define CMDQ_ATC_0_SID GENMASK_ULL(63, 32)
338 #define CMDQ_ATC_0_GLOBAL (1UL << 9)
339 #define CMDQ_ATC_1_SIZE GENMASK_ULL(5, 0)
340 #define CMDQ_ATC_1_ADDR_MASK GENMASK_ULL(63, 12)
342 #define CMDQ_PRI_0_SSID GENMASK_ULL(31, 12)
343 #define CMDQ_PRI_0_SID GENMASK_ULL(63, 32)
344 #define CMDQ_PRI_1_GRPID GENMASK_ULL(8, 0)
345 #define CMDQ_PRI_1_RESP GENMASK_ULL(13, 12)
347 #define CMDQ_SYNC_0_CS GENMASK_ULL(13, 12)
348 #define CMDQ_SYNC_0_CS_NONE 0
349 #define CMDQ_SYNC_0_CS_IRQ 1
350 #define CMDQ_SYNC_0_CS_SEV 2
351 #define CMDQ_SYNC_0_MSH GENMASK_ULL(23, 22)
352 #define CMDQ_SYNC_0_MSIATTR GENMASK_ULL(27, 24)
353 #define CMDQ_SYNC_0_MSIDATA GENMASK_ULL(63, 32)
354 #define CMDQ_SYNC_1_MSIADDR_MASK GENMASK_ULL(51, 2)
357 #define EVTQ_ENT_SZ_SHIFT 5
358 #define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
359 #define EVTQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
361 #define EVTQ_0_ID GENMASK_ULL(7, 0)
364 #define PRIQ_ENT_SZ_SHIFT 4
365 #define PRIQ_ENT_DWORDS ((1 << PRIQ_ENT_SZ_SHIFT) >> 3)
366 #define PRIQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
368 #define PRIQ_0_SID GENMASK_ULL(31, 0)
369 #define PRIQ_0_SSID GENMASK_ULL(51, 32)
370 #define PRIQ_0_PERM_PRIV (1UL << 58)
371 #define PRIQ_0_PERM_EXEC (1UL << 59)
372 #define PRIQ_0_PERM_READ (1UL << 60)
373 #define PRIQ_0_PERM_WRITE (1UL << 61)
374 #define PRIQ_0_PRG_LAST (1UL << 62)
375 #define PRIQ_0_SSID_V (1UL << 63)
377 #define PRIQ_1_PRG_IDX GENMASK_ULL(8, 0)
378 #define PRIQ_1_ADDR_MASK GENMASK_ULL(63, 12)
380 /* High-level queue structures */
381 #define ARM_SMMU_POLL_TIMEOUT_US 1000000 /* 1s! */
382 #define ARM_SMMU_POLL_SPIN_COUNT 10
384 #define MSI_IOVA_BASE 0x8000000
385 #define MSI_IOVA_LENGTH 0x100000
388 * not really modular, but the easiest way to keep compat with existing
389 * bootargs behaviour is to continue using module_param_named here.
391 static bool disable_bypass = 1;
392 module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO);
393 MODULE_PARM_DESC(disable_bypass,
394 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
402 enum arm_smmu_msi_index {
409 static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
411 ARM_SMMU_EVTQ_IRQ_CFG0,
412 ARM_SMMU_EVTQ_IRQ_CFG1,
413 ARM_SMMU_EVTQ_IRQ_CFG2,
415 [GERROR_MSI_INDEX] = {
416 ARM_SMMU_GERROR_IRQ_CFG0,
417 ARM_SMMU_GERROR_IRQ_CFG1,
418 ARM_SMMU_GERROR_IRQ_CFG2,
421 ARM_SMMU_PRIQ_IRQ_CFG0,
422 ARM_SMMU_PRIQ_IRQ_CFG1,
423 ARM_SMMU_PRIQ_IRQ_CFG2,
427 struct arm_smmu_cmdq_ent {
430 bool substream_valid;
432 /* Command-specific fields */
434 #define CMDQ_OP_PREFETCH_CFG 0x1
441 #define CMDQ_OP_CFGI_STE 0x3
442 #define CMDQ_OP_CFGI_ALL 0x4
451 #define CMDQ_OP_TLBI_NH_ASID 0x11
452 #define CMDQ_OP_TLBI_NH_VA 0x12
453 #define CMDQ_OP_TLBI_EL2_ALL 0x20
454 #define CMDQ_OP_TLBI_S12_VMALL 0x28
455 #define CMDQ_OP_TLBI_S2_IPA 0x2a
456 #define CMDQ_OP_TLBI_NSNH_ALL 0x30
464 #define CMDQ_OP_ATC_INV 0x40
465 #define ATC_INV_SIZE_ALL 52
474 #define CMDQ_OP_PRI_RESP 0x41
482 #define CMDQ_OP_CMD_SYNC 0x46
489 struct arm_smmu_ll_queue {
500 u8 __pad[SMP_CACHE_BYTES];
501 } ____cacheline_aligned_in_smp;
505 struct arm_smmu_queue {
506 struct arm_smmu_ll_queue llq;
507 int irq; /* Wired interrupt */
515 u32 __iomem *prod_reg;
516 u32 __iomem *cons_reg;
519 struct arm_smmu_queue_poll {
522 unsigned int spin_cnt;
526 struct arm_smmu_cmdq {
527 struct arm_smmu_queue q;
528 atomic_long_t *valid_map;
533 struct arm_smmu_evtq {
534 struct arm_smmu_queue q;
538 struct arm_smmu_priq {
539 struct arm_smmu_queue q;
542 /* High-level stream table and context descriptor structures */
543 struct arm_smmu_strtab_l1_desc {
547 dma_addr_t l2ptr_dma;
550 struct arm_smmu_s1_cfg {
552 dma_addr_t cdptr_dma;
554 struct arm_smmu_ctx_desc {
562 struct arm_smmu_s2_cfg {
568 struct arm_smmu_strtab_cfg {
570 dma_addr_t strtab_dma;
571 struct arm_smmu_strtab_l1_desc *l1_desc;
572 unsigned int num_l1_ents;
578 /* An SMMUv3 instance */
579 struct arm_smmu_device {
583 #define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0)
584 #define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1)
585 #define ARM_SMMU_FEAT_TT_LE (1 << 2)
586 #define ARM_SMMU_FEAT_TT_BE (1 << 3)
587 #define ARM_SMMU_FEAT_PRI (1 << 4)
588 #define ARM_SMMU_FEAT_ATS (1 << 5)
589 #define ARM_SMMU_FEAT_SEV (1 << 6)
590 #define ARM_SMMU_FEAT_MSI (1 << 7)
591 #define ARM_SMMU_FEAT_COHERENCY (1 << 8)
592 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 9)
593 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 10)
594 #define ARM_SMMU_FEAT_STALLS (1 << 11)
595 #define ARM_SMMU_FEAT_HYP (1 << 12)
596 #define ARM_SMMU_FEAT_STALL_FORCE (1 << 13)
597 #define ARM_SMMU_FEAT_VAX (1 << 14)
600 #define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0)
601 #define ARM_SMMU_OPT_PAGE0_REGS_ONLY (1 << 1)
604 struct arm_smmu_cmdq cmdq;
605 struct arm_smmu_evtq evtq;
606 struct arm_smmu_priq priq;
611 unsigned long ias; /* IPA */
612 unsigned long oas; /* PA */
613 unsigned long pgsize_bitmap;
615 #define ARM_SMMU_MAX_ASIDS (1 << 16)
616 unsigned int asid_bits;
617 DECLARE_BITMAP(asid_map, ARM_SMMU_MAX_ASIDS);
619 #define ARM_SMMU_MAX_VMIDS (1 << 16)
620 unsigned int vmid_bits;
621 DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
623 unsigned int ssid_bits;
624 unsigned int sid_bits;
626 struct arm_smmu_strtab_cfg strtab_cfg;
628 /* IOMMU core code handle */
629 struct iommu_device iommu;
632 /* SMMU private data for each master */
633 struct arm_smmu_master {
634 struct arm_smmu_device *smmu;
636 struct arm_smmu_domain *domain;
637 struct list_head domain_head;
639 unsigned int num_sids;
643 /* SMMU private data for an IOMMU domain */
644 enum arm_smmu_domain_stage {
645 ARM_SMMU_DOMAIN_S1 = 0,
647 ARM_SMMU_DOMAIN_NESTED,
648 ARM_SMMU_DOMAIN_BYPASS,
651 struct arm_smmu_domain {
652 struct arm_smmu_device *smmu;
653 struct mutex init_mutex; /* Protects smmu pointer */
655 struct io_pgtable_ops *pgtbl_ops;
657 atomic_t nr_ats_masters;
659 enum arm_smmu_domain_stage stage;
661 struct arm_smmu_s1_cfg s1_cfg;
662 struct arm_smmu_s2_cfg s2_cfg;
665 struct iommu_domain domain;
667 struct list_head devices;
668 spinlock_t devices_lock;
671 struct arm_smmu_option_prop {
676 static struct arm_smmu_option_prop arm_smmu_options[] = {
677 { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
678 { ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"},
682 static inline void __iomem *arm_smmu_page1_fixup(unsigned long offset,
683 struct arm_smmu_device *smmu)
685 if ((offset > SZ_64K) &&
686 (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY))
689 return smmu->base + offset;
692 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
694 return container_of(dom, struct arm_smmu_domain, domain);
697 static void parse_driver_options(struct arm_smmu_device *smmu)
702 if (of_property_read_bool(smmu->dev->of_node,
703 arm_smmu_options[i].prop)) {
704 smmu->options |= arm_smmu_options[i].opt;
705 dev_notice(smmu->dev, "option %s\n",
706 arm_smmu_options[i].prop);
708 } while (arm_smmu_options[++i].opt);
711 /* Low-level queue manipulation functions */
712 static bool queue_has_space(struct arm_smmu_ll_queue *q, u32 n)
714 u32 space, prod, cons;
716 prod = Q_IDX(q, q->prod);
717 cons = Q_IDX(q, q->cons);
719 if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons))
720 space = (1 << q->max_n_shift) - (prod - cons);
727 static bool queue_full(struct arm_smmu_ll_queue *q)
729 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
730 Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
733 static bool queue_empty(struct arm_smmu_ll_queue *q)
735 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
736 Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
739 static bool queue_consumed(struct arm_smmu_ll_queue *q, u32 prod)
741 return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) &&
742 (Q_IDX(q, q->cons) > Q_IDX(q, prod))) ||
743 ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) &&
744 (Q_IDX(q, q->cons) <= Q_IDX(q, prod)));
747 static void queue_sync_cons_out(struct arm_smmu_queue *q)
750 * Ensure that all CPU accesses (reads and writes) to the queue
751 * are complete before we update the cons pointer.
754 writel_relaxed(q->llq.cons, q->cons_reg);
757 static void queue_inc_cons(struct arm_smmu_ll_queue *q)
759 u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
760 q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
763 static int queue_sync_prod_in(struct arm_smmu_queue *q)
766 u32 prod = readl_relaxed(q->prod_reg);
768 if (Q_OVF(prod) != Q_OVF(q->llq.prod))
775 static u32 queue_inc_prod_n(struct arm_smmu_ll_queue *q, int n)
777 u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n;
778 return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
781 static void queue_poll_init(struct arm_smmu_device *smmu,
782 struct arm_smmu_queue_poll *qp)
786 qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
787 qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
790 static int queue_poll(struct arm_smmu_queue_poll *qp)
792 if (ktime_compare(ktime_get(), qp->timeout) > 0)
797 } else if (++qp->spin_cnt < ARM_SMMU_POLL_SPIN_COUNT) {
808 static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
812 for (i = 0; i < n_dwords; ++i)
813 *dst++ = cpu_to_le64(*src++);
816 static void queue_read(__le64 *dst, u64 *src, size_t n_dwords)
820 for (i = 0; i < n_dwords; ++i)
821 *dst++ = le64_to_cpu(*src++);
824 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
826 if (queue_empty(&q->llq))
829 queue_read(ent, Q_ENT(q, q->llq.cons), q->ent_dwords);
830 queue_inc_cons(&q->llq);
831 queue_sync_cons_out(q);
835 /* High-level queue accessors */
836 static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
838 memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT);
839 cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
841 switch (ent->opcode) {
842 case CMDQ_OP_TLBI_EL2_ALL:
843 case CMDQ_OP_TLBI_NSNH_ALL:
845 case CMDQ_OP_PREFETCH_CFG:
846 cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid);
847 cmd[1] |= FIELD_PREP(CMDQ_PREFETCH_1_SIZE, ent->prefetch.size);
848 cmd[1] |= ent->prefetch.addr & CMDQ_PREFETCH_1_ADDR_MASK;
850 case CMDQ_OP_CFGI_STE:
851 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
852 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);
854 case CMDQ_OP_CFGI_ALL:
855 /* Cover the entire SID range */
856 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
858 case CMDQ_OP_TLBI_NH_VA:
859 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
860 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
861 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
862 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
864 case CMDQ_OP_TLBI_S2_IPA:
865 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
866 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
867 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
869 case CMDQ_OP_TLBI_NH_ASID:
870 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
872 case CMDQ_OP_TLBI_S12_VMALL:
873 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
875 case CMDQ_OP_ATC_INV:
876 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
877 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);
878 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid);
879 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid);
880 cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);
881 cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;
883 case CMDQ_OP_PRI_RESP:
884 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
885 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid);
886 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid);
887 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid);
888 switch (ent->pri.resp) {
896 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp);
898 case CMDQ_OP_CMD_SYNC:
899 if (ent->sync.msiaddr) {
900 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ);
901 cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
903 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV);
905 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH);
906 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB);
915 static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu,
918 struct arm_smmu_queue *q = &smmu->cmdq.q;
919 struct arm_smmu_cmdq_ent ent = {
920 .opcode = CMDQ_OP_CMD_SYNC,
924 * Beware that Hi16xx adds an extra 32 bits of goodness to its MSI
925 * payload, so the write will zero the entire command on that platform.
927 if (smmu->features & ARM_SMMU_FEAT_MSI &&
928 smmu->features & ARM_SMMU_FEAT_COHERENCY) {
929 ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *
933 arm_smmu_cmdq_build_cmd(cmd, &ent);
936 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
938 static const char *cerror_str[] = {
939 [CMDQ_ERR_CERROR_NONE_IDX] = "No error",
940 [CMDQ_ERR_CERROR_ILL_IDX] = "Illegal command",
941 [CMDQ_ERR_CERROR_ABT_IDX] = "Abort on command fetch",
942 [CMDQ_ERR_CERROR_ATC_INV_IDX] = "ATC invalidate timeout",
946 u64 cmd[CMDQ_ENT_DWORDS];
947 struct arm_smmu_queue *q = &smmu->cmdq.q;
948 u32 cons = readl_relaxed(q->cons_reg);
949 u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);
950 struct arm_smmu_cmdq_ent cmd_sync = {
951 .opcode = CMDQ_OP_CMD_SYNC,
954 dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
955 idx < ARRAY_SIZE(cerror_str) ? cerror_str[idx] : "Unknown");
958 case CMDQ_ERR_CERROR_ABT_IDX:
959 dev_err(smmu->dev, "retrying command fetch\n");
960 case CMDQ_ERR_CERROR_NONE_IDX:
962 case CMDQ_ERR_CERROR_ATC_INV_IDX:
964 * ATC Invalidation Completion timeout. CONS is still pointing
965 * at the CMD_SYNC. Attempt to complete other pending commands
966 * by repeating the CMD_SYNC, though we might well end up back
967 * here since the ATC invalidation may still be pending.
970 case CMDQ_ERR_CERROR_ILL_IDX:
977 * We may have concurrent producers, so we need to be careful
978 * not to touch any of the shadow cmdq state.
980 queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
981 dev_err(smmu->dev, "skipping command in error state:\n");
982 for (i = 0; i < ARRAY_SIZE(cmd); ++i)
983 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
985 /* Convert the erroneous command into a CMD_SYNC */
986 if (arm_smmu_cmdq_build_cmd(cmd, &cmd_sync)) {
987 dev_err(smmu->dev, "failed to convert to CMD_SYNC\n");
991 queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
995 * Command queue locking.
996 * This is a form of bastardised rwlock with the following major changes:
998 * - The only LOCK routines are exclusive_trylock() and shared_lock().
999 * Neither have barrier semantics, and instead provide only a control
1002 * - The UNLOCK routines are supplemented with shared_tryunlock(), which
1003 * fails if the caller appears to be the last lock holder (yes, this is
1004 * racy). All successful UNLOCK routines have RELEASE semantics.
1006 static void arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq *cmdq)
1011 * We can try to avoid the cmpxchg() loop by simply incrementing the
1012 * lock counter. When held in exclusive state, the lock counter is set
1013 * to INT_MIN so these increments won't hurt as the value will remain
1016 if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0)
1020 val = atomic_cond_read_relaxed(&cmdq->lock, VAL >= 0);
1021 } while (atomic_cmpxchg_relaxed(&cmdq->lock, val, val + 1) != val);
1024 static void arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq *cmdq)
1026 (void)atomic_dec_return_release(&cmdq->lock);
1029 static bool arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq *cmdq)
1031 if (atomic_read(&cmdq->lock) == 1)
1034 arm_smmu_cmdq_shared_unlock(cmdq);
1038 #define arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags) \
1041 local_irq_save(flags); \
1042 __ret = !atomic_cmpxchg_relaxed(&cmdq->lock, 0, INT_MIN); \
1044 local_irq_restore(flags); \
1048 #define arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags) \
1050 atomic_set_release(&cmdq->lock, 0); \
1051 local_irq_restore(flags); \
1056 * Command queue insertion.
1057 * This is made fiddly by our attempts to achieve some sort of scalability
1058 * since there is one queue shared amongst all of the CPUs in the system. If
1059 * you like mixed-size concurrency, dependency ordering and relaxed atomics,
1060 * then you'll *love* this monstrosity.
1062 * The basic idea is to split the queue up into ranges of commands that are
1063 * owned by a given CPU; the owner may not have written all of the commands
1064 * itself, but is responsible for advancing the hardware prod pointer when
1065 * the time comes. The algorithm is roughly:
1067 * 1. Allocate some space in the queue. At this point we also discover
1068 * whether the head of the queue is currently owned by another CPU,
1069 * or whether we are the owner.
1071 * 2. Write our commands into our allocated slots in the queue.
1073 * 3. Mark our slots as valid in arm_smmu_cmdq.valid_map.
1075 * 4. If we are an owner:
1076 * a. Wait for the previous owner to finish.
1077 * b. Mark the queue head as unowned, which tells us the range
1078 * that we are responsible for publishing.
1079 * c. Wait for all commands in our owned range to become valid.
1080 * d. Advance the hardware prod pointer.
1081 * e. Tell the next owner we've finished.
1083 * 5. If we are inserting a CMD_SYNC (we may or may not have been an
1084 * owner), then we need to stick around until it has completed:
1085 * a. If we have MSIs, the SMMU can write back into the CMD_SYNC
1086 * to clear the first 4 bytes.
1087 * b. Otherwise, we spin waiting for the hardware cons pointer to
1088 * advance past our command.
1090 * The devil is in the details, particularly the use of locking for handling
1091 * SYNC completion and freeing up space in the queue before we think that it is
1094 static void __arm_smmu_cmdq_poll_set_valid_map(struct arm_smmu_cmdq *cmdq,
1095 u32 sprod, u32 eprod, bool set)
1097 u32 swidx, sbidx, ewidx, ebidx;
1098 struct arm_smmu_ll_queue llq = {
1099 .max_n_shift = cmdq->q.llq.max_n_shift,
1103 ewidx = BIT_WORD(Q_IDX(&llq, eprod));
1104 ebidx = Q_IDX(&llq, eprod) % BITS_PER_LONG;
1106 while (llq.prod != eprod) {
1109 u32 limit = BITS_PER_LONG;
1111 swidx = BIT_WORD(Q_IDX(&llq, llq.prod));
1112 sbidx = Q_IDX(&llq, llq.prod) % BITS_PER_LONG;
1114 ptr = &cmdq->valid_map[swidx];
1116 if ((swidx == ewidx) && (sbidx < ebidx))
1119 mask = GENMASK(limit - 1, sbidx);
1122 * The valid bit is the inverse of the wrap bit. This means
1123 * that a zero-initialised queue is invalid and, after marking
1124 * all entries as valid, they become invalid again when we
1128 atomic_long_xor(mask, ptr);
1130 unsigned long valid;
1132 valid = (ULONG_MAX + !!Q_WRP(&llq, llq.prod)) & mask;
1133 atomic_long_cond_read_relaxed(ptr, (VAL & mask) == valid);
1136 llq.prod = queue_inc_prod_n(&llq, limit - sbidx);
1140 /* Mark all entries in the range [sprod, eprod) as valid */
1141 static void arm_smmu_cmdq_set_valid_map(struct arm_smmu_cmdq *cmdq,
1142 u32 sprod, u32 eprod)
1144 __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, true);
1147 /* Wait for all entries in the range [sprod, eprod) to become valid */
1148 static void arm_smmu_cmdq_poll_valid_map(struct arm_smmu_cmdq *cmdq,
1149 u32 sprod, u32 eprod)
1151 __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, false);
1154 /* Wait for the command queue to become non-full */
1155 static int arm_smmu_cmdq_poll_until_not_full(struct arm_smmu_device *smmu,
1156 struct arm_smmu_ll_queue *llq)
1158 unsigned long flags;
1159 struct arm_smmu_queue_poll qp;
1160 struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
1164 * Try to update our copy of cons by grabbing exclusive cmdq access. If
1165 * that fails, spin until somebody else updates it for us.
1167 if (arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)) {
1168 WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg));
1169 arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags);
1170 llq->val = READ_ONCE(cmdq->q.llq.val);
1174 queue_poll_init(smmu, &qp);
1176 llq->val = READ_ONCE(smmu->cmdq.q.llq.val);
1177 if (!queue_full(llq))
1180 ret = queue_poll(&qp);
1187 * Wait until the SMMU signals a CMD_SYNC completion MSI.
1188 * Must be called with the cmdq lock held in some capacity.
1190 static int __arm_smmu_cmdq_poll_until_msi(struct arm_smmu_device *smmu,
1191 struct arm_smmu_ll_queue *llq)
1194 struct arm_smmu_queue_poll qp;
1195 struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
1196 u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod));
1198 queue_poll_init(smmu, &qp);
1201 * The MSI won't generate an event, since it's being written back
1202 * into the command queue.
1205 smp_cond_load_relaxed(cmd, !VAL || (ret = queue_poll(&qp)));
1206 llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1);
1211 * Wait until the SMMU cons index passes llq->prod.
1212 * Must be called with the cmdq lock held in some capacity.
1214 static int __arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device *smmu,
1215 struct arm_smmu_ll_queue *llq)
1217 struct arm_smmu_queue_poll qp;
1218 struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
1219 u32 prod = llq->prod;
1222 queue_poll_init(smmu, &qp);
1223 llq->val = READ_ONCE(smmu->cmdq.q.llq.val);
1225 if (queue_consumed(llq, prod))
1228 ret = queue_poll(&qp);
1231 * This needs to be a readl() so that our subsequent call
1232 * to arm_smmu_cmdq_shared_tryunlock() can fail accurately.
1234 * Specifically, we need to ensure that we observe all
1235 * shared_lock()s by other CMD_SYNCs that share our owner,
1236 * so that a failing call to tryunlock() means that we're
1237 * the last one out and therefore we can safely advance
1238 * cmdq->q.llq.cons. Roughly speaking:
1240 * CPU 0 CPU1 CPU2 (us)
1250 * <control dependency>
1256 * Requires us to see CPU 0's shared_lock() acquisition.
1258 llq->cons = readl(cmdq->q.cons_reg);
1264 static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu,
1265 struct arm_smmu_ll_queue *llq)
1267 if (smmu->features & ARM_SMMU_FEAT_MSI &&
1268 smmu->features & ARM_SMMU_FEAT_COHERENCY)
1269 return __arm_smmu_cmdq_poll_until_msi(smmu, llq);
1271 return __arm_smmu_cmdq_poll_until_consumed(smmu, llq);
1274 static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds,
1278 struct arm_smmu_ll_queue llq = {
1279 .max_n_shift = cmdq->q.llq.max_n_shift,
1283 for (i = 0; i < n; ++i) {
1284 u64 *cmd = &cmds[i * CMDQ_ENT_DWORDS];
1286 prod = queue_inc_prod_n(&llq, i);
1287 queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS);
1292 * This is the actual insertion function, and provides the following
1293 * ordering guarantees to callers:
1295 * - There is a dma_wmb() before publishing any commands to the queue.
1296 * This can be relied upon to order prior writes to data structures
1297 * in memory (such as a CD or an STE) before the command.
1299 * - On completion of a CMD_SYNC, there is a control dependency.
1300 * This can be relied upon to order subsequent writes to memory (e.g.
1301 * freeing an IOVA) after completion of the CMD_SYNC.
1303 * - Command insertion is totally ordered, so if two CPUs each race to
1304 * insert their own list of commands then all of the commands from one
1305 * CPU will appear before any of the commands from the other CPU.
1307 static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
1308 u64 *cmds, int n, bool sync)
1310 u64 cmd_sync[CMDQ_ENT_DWORDS];
1312 unsigned long flags;
1314 struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
1315 struct arm_smmu_ll_queue llq = {
1316 .max_n_shift = cmdq->q.llq.max_n_shift,
1320 /* 1. Allocate some space in the queue */
1321 local_irq_save(flags);
1322 llq.val = READ_ONCE(cmdq->q.llq.val);
1326 while (!queue_has_space(&llq, n + sync)) {
1327 local_irq_restore(flags);
1328 if (arm_smmu_cmdq_poll_until_not_full(smmu, &llq))
1329 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
1330 local_irq_save(flags);
1333 head.cons = llq.cons;
1334 head.prod = queue_inc_prod_n(&llq, n + sync) |
1335 CMDQ_PROD_OWNED_FLAG;
1337 old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val);
1343 owner = !(llq.prod & CMDQ_PROD_OWNED_FLAG);
1344 head.prod &= ~CMDQ_PROD_OWNED_FLAG;
1345 llq.prod &= ~CMDQ_PROD_OWNED_FLAG;
1348 * 2. Write our commands into the queue
1349 * Dependency ordering from the cmpxchg() loop above.
1351 arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);
1353 prod = queue_inc_prod_n(&llq, n);
1354 arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, prod);
1355 queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS);
1358 * In order to determine completion of our CMD_SYNC, we must
1359 * ensure that the queue can't wrap twice without us noticing.
1360 * We achieve that by taking the cmdq lock as shared before
1361 * marking our slot as valid.
1363 arm_smmu_cmdq_shared_lock(cmdq);
1366 /* 3. Mark our slots as valid, ensuring commands are visible first */
1368 arm_smmu_cmdq_set_valid_map(cmdq, llq.prod, head.prod);
1370 /* 4. If we are the owner, take control of the SMMU hardware */
1372 /* a. Wait for previous owner to finish */
1373 atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod);
1375 /* b. Stop gathering work by clearing the owned flag */
1376 prod = atomic_fetch_andnot_relaxed(CMDQ_PROD_OWNED_FLAG,
1377 &cmdq->q.llq.atomic.prod);
1378 prod &= ~CMDQ_PROD_OWNED_FLAG;
1381 * c. Wait for any gathered work to be written to the queue.
1382 * Note that we read our own entries so that we have the control
1383 * dependency required by (d).
1385 arm_smmu_cmdq_poll_valid_map(cmdq, llq.prod, prod);
1388 * d. Advance the hardware prod pointer
1389 * Control dependency ordering from the entries becoming valid.
1391 writel_relaxed(prod, cmdq->q.prod_reg);
1394 * e. Tell the next owner we're done
1395 * Make sure we've updated the hardware first, so that we don't
1396 * race to update prod and potentially move it backwards.
1398 atomic_set_release(&cmdq->owner_prod, prod);
1401 /* 5. If we are inserting a CMD_SYNC, we must wait for it to complete */
1403 llq.prod = queue_inc_prod_n(&llq, n);
1404 ret = arm_smmu_cmdq_poll_until_sync(smmu, &llq);
1406 dev_err_ratelimited(smmu->dev,
1407 "CMD_SYNC timeout at 0x%08x [hwprod 0x%08x, hwcons 0x%08x]\n",
1409 readl_relaxed(cmdq->q.prod_reg),
1410 readl_relaxed(cmdq->q.cons_reg));
1414 * Try to unlock the cmq lock. This will fail if we're the last
1415 * reader, in which case we can safely update cmdq->q.llq.cons
1417 if (!arm_smmu_cmdq_shared_tryunlock(cmdq)) {
1418 WRITE_ONCE(cmdq->q.llq.cons, llq.cons);
1419 arm_smmu_cmdq_shared_unlock(cmdq);
1423 local_irq_restore(flags);
1427 static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
1428 struct arm_smmu_cmdq_ent *ent)
1430 u64 cmd[CMDQ_ENT_DWORDS];
1432 if (arm_smmu_cmdq_build_cmd(cmd, ent)) {
1433 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
1438 return arm_smmu_cmdq_issue_cmdlist(smmu, cmd, 1, false);
1441 static int arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
1443 return arm_smmu_cmdq_issue_cmdlist(smmu, NULL, 0, true);
1446 /* Context descriptor manipulation functions */
1447 static u64 arm_smmu_cpu_tcr_to_cd(u64 tcr)
1451 /* Repack the TCR. Just care about TTBR0 for now */
1452 val |= ARM_SMMU_TCR2CD(tcr, T0SZ);
1453 val |= ARM_SMMU_TCR2CD(tcr, TG0);
1454 val |= ARM_SMMU_TCR2CD(tcr, IRGN0);
1455 val |= ARM_SMMU_TCR2CD(tcr, ORGN0);
1456 val |= ARM_SMMU_TCR2CD(tcr, SH0);
1457 val |= ARM_SMMU_TCR2CD(tcr, EPD0);
1458 val |= ARM_SMMU_TCR2CD(tcr, EPD1);
1459 val |= ARM_SMMU_TCR2CD(tcr, IPS);
1464 static void arm_smmu_write_ctx_desc(struct arm_smmu_device *smmu,
1465 struct arm_smmu_s1_cfg *cfg)
1470 * We don't need to issue any invalidation here, as we'll invalidate
1471 * the STE when installing the new entry anyway.
1473 val = arm_smmu_cpu_tcr_to_cd(cfg->cd.tcr) |
1477 CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET |
1478 CTXDESC_CD_0_AA64 | FIELD_PREP(CTXDESC_CD_0_ASID, cfg->cd.asid) |
1481 /* STALL_MODEL==0b10 && CD.S==0 is ILLEGAL */
1482 if (smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
1483 val |= CTXDESC_CD_0_S;
1485 cfg->cdptr[0] = cpu_to_le64(val);
1487 val = cfg->cd.ttbr & CTXDESC_CD_1_TTB0_MASK;
1488 cfg->cdptr[1] = cpu_to_le64(val);
1490 cfg->cdptr[3] = cpu_to_le64(cfg->cd.mair);
1493 /* Stream table manipulation functions */
1495 arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc)
1499 val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, desc->span);
1500 val |= desc->l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK;
1502 *dst = cpu_to_le64(val);
1505 static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid)
1507 struct arm_smmu_cmdq_ent cmd = {
1508 .opcode = CMDQ_OP_CFGI_STE,
1515 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1516 arm_smmu_cmdq_issue_sync(smmu);
1519 static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
1523 * This is hideously complicated, but we only really care about
1524 * three cases at the moment:
1526 * 1. Invalid (all zero) -> bypass/fault (init)
1527 * 2. Bypass/fault -> translation/bypass (attach)
1528 * 3. Translation/bypass -> bypass/fault (detach)
1530 * Given that we can't update the STE atomically and the SMMU
1531 * doesn't read the thing in a defined order, that leaves us
1532 * with the following maintenance requirements:
1534 * 1. Update Config, return (init time STEs aren't live)
1535 * 2. Write everything apart from dword 0, sync, write dword 0, sync
1536 * 3. Update Config, sync
1538 u64 val = le64_to_cpu(dst[0]);
1539 bool ste_live = false;
1540 struct arm_smmu_device *smmu = NULL;
1541 struct arm_smmu_s1_cfg *s1_cfg = NULL;
1542 struct arm_smmu_s2_cfg *s2_cfg = NULL;
1543 struct arm_smmu_domain *smmu_domain = NULL;
1544 struct arm_smmu_cmdq_ent prefetch_cmd = {
1545 .opcode = CMDQ_OP_PREFETCH_CFG,
1552 smmu_domain = master->domain;
1553 smmu = master->smmu;
1557 switch (smmu_domain->stage) {
1558 case ARM_SMMU_DOMAIN_S1:
1559 s1_cfg = &smmu_domain->s1_cfg;
1561 case ARM_SMMU_DOMAIN_S2:
1562 case ARM_SMMU_DOMAIN_NESTED:
1563 s2_cfg = &smmu_domain->s2_cfg;
1570 if (val & STRTAB_STE_0_V) {
1571 switch (FIELD_GET(STRTAB_STE_0_CFG, val)) {
1572 case STRTAB_STE_0_CFG_BYPASS:
1574 case STRTAB_STE_0_CFG_S1_TRANS:
1575 case STRTAB_STE_0_CFG_S2_TRANS:
1578 case STRTAB_STE_0_CFG_ABORT:
1579 BUG_ON(!disable_bypass);
1582 BUG(); /* STE corruption */
1586 /* Nuke the existing STE_0 value, as we're going to rewrite it */
1587 val = STRTAB_STE_0_V;
1590 if (!smmu_domain || !(s1_cfg || s2_cfg)) {
1591 if (!smmu_domain && disable_bypass)
1592 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT);
1594 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS);
1596 dst[0] = cpu_to_le64(val);
1597 dst[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1598 STRTAB_STE_1_SHCFG_INCOMING));
1599 dst[2] = 0; /* Nuke the VMID */
1601 * The SMMU can perform negative caching, so we must sync
1602 * the STE regardless of whether the old value was live.
1605 arm_smmu_sync_ste_for_sid(smmu, sid);
1611 dst[1] = cpu_to_le64(
1612 FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1613 FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1614 FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) |
1615 FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_NSEL1));
1617 if (smmu->features & ARM_SMMU_FEAT_STALLS &&
1618 !(smmu->features & ARM_SMMU_FEAT_STALL_FORCE))
1619 dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
1621 val |= (s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK) |
1622 FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS);
1627 dst[2] = cpu_to_le64(
1628 FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) |
1629 FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) |
1631 STRTAB_STE_2_S2ENDI |
1633 STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 |
1636 dst[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK);
1638 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS);
1641 if (master->ats_enabled)
1642 dst[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_EATS,
1643 STRTAB_STE_1_EATS_TRANS));
1645 arm_smmu_sync_ste_for_sid(smmu, sid);
1646 /* See comment in arm_smmu_write_ctx_desc() */
1647 WRITE_ONCE(dst[0], cpu_to_le64(val));
1648 arm_smmu_sync_ste_for_sid(smmu, sid);
1650 /* It's likely that we'll want to use the new STE soon */
1651 if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH))
1652 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
1655 static void arm_smmu_init_bypass_stes(u64 *strtab, unsigned int nent)
1659 for (i = 0; i < nent; ++i) {
1660 arm_smmu_write_strtab_ent(NULL, -1, strtab);
1661 strtab += STRTAB_STE_DWORDS;
1665 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
1669 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1670 struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT];
1675 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
1676 strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS];
1678 desc->span = STRTAB_SPLIT + 1;
1679 desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
1680 GFP_KERNEL | __GFP_ZERO);
1683 "failed to allocate l2 stream table for SID %u\n",
1688 arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT);
1689 arm_smmu_write_strtab_l1_desc(strtab, desc);
1693 /* IRQ and event handlers */
1694 static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
1697 struct arm_smmu_device *smmu = dev;
1698 struct arm_smmu_queue *q = &smmu->evtq.q;
1699 struct arm_smmu_ll_queue *llq = &q->llq;
1700 u64 evt[EVTQ_ENT_DWORDS];
1703 while (!queue_remove_raw(q, evt)) {
1704 u8 id = FIELD_GET(EVTQ_0_ID, evt[0]);
1706 dev_info(smmu->dev, "event 0x%02x received:\n", id);
1707 for (i = 0; i < ARRAY_SIZE(evt); ++i)
1708 dev_info(smmu->dev, "\t0x%016llx\n",
1709 (unsigned long long)evt[i]);
1715 * Not much we can do on overflow, so scream and pretend we're
1718 if (queue_sync_prod_in(q) == -EOVERFLOW)
1719 dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
1720 } while (!queue_empty(llq));
1722 /* Sync our overflow flag, as we believe we're up to speed */
1723 llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |
1724 Q_IDX(llq, llq->cons);
1728 static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
1734 sid = FIELD_GET(PRIQ_0_SID, evt[0]);
1735 ssv = FIELD_GET(PRIQ_0_SSID_V, evt[0]);
1736 ssid = ssv ? FIELD_GET(PRIQ_0_SSID, evt[0]) : 0;
1737 last = FIELD_GET(PRIQ_0_PRG_LAST, evt[0]);
1738 grpid = FIELD_GET(PRIQ_1_PRG_IDX, evt[1]);
1740 dev_info(smmu->dev, "unexpected PRI request received:\n");
1742 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
1743 sid, ssid, grpid, last ? "L" : "",
1744 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
1745 evt[0] & PRIQ_0_PERM_READ ? "R" : "",
1746 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
1747 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
1748 evt[1] & PRIQ_1_ADDR_MASK);
1751 struct arm_smmu_cmdq_ent cmd = {
1752 .opcode = CMDQ_OP_PRI_RESP,
1753 .substream_valid = ssv,
1758 .resp = PRI_RESP_DENY,
1762 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1766 static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
1768 struct arm_smmu_device *smmu = dev;
1769 struct arm_smmu_queue *q = &smmu->priq.q;
1770 struct arm_smmu_ll_queue *llq = &q->llq;
1771 u64 evt[PRIQ_ENT_DWORDS];
1774 while (!queue_remove_raw(q, evt))
1775 arm_smmu_handle_ppr(smmu, evt);
1777 if (queue_sync_prod_in(q) == -EOVERFLOW)
1778 dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
1779 } while (!queue_empty(llq));
1781 /* Sync our overflow flag, as we believe we're up to speed */
1782 llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |
1783 Q_IDX(llq, llq->cons);
1784 queue_sync_cons_out(q);
1788 static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
1790 static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
1792 u32 gerror, gerrorn, active;
1793 struct arm_smmu_device *smmu = dev;
1795 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
1796 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
1798 active = gerror ^ gerrorn;
1799 if (!(active & GERROR_ERR_MASK))
1800 return IRQ_NONE; /* No errors pending */
1803 "unexpected global error reported (0x%08x), this could be serious\n",
1806 if (active & GERROR_SFM_ERR) {
1807 dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
1808 arm_smmu_device_disable(smmu);
1811 if (active & GERROR_MSI_GERROR_ABT_ERR)
1812 dev_warn(smmu->dev, "GERROR MSI write aborted\n");
1814 if (active & GERROR_MSI_PRIQ_ABT_ERR)
1815 dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
1817 if (active & GERROR_MSI_EVTQ_ABT_ERR)
1818 dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
1820 if (active & GERROR_MSI_CMDQ_ABT_ERR)
1821 dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
1823 if (active & GERROR_PRIQ_ABT_ERR)
1824 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
1826 if (active & GERROR_EVTQ_ABT_ERR)
1827 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
1829 if (active & GERROR_CMDQ_ERR)
1830 arm_smmu_cmdq_skip_err(smmu);
1832 writel(gerror, smmu->base + ARM_SMMU_GERRORN);
1836 static irqreturn_t arm_smmu_combined_irq_thread(int irq, void *dev)
1838 struct arm_smmu_device *smmu = dev;
1840 arm_smmu_evtq_thread(irq, dev);
1841 if (smmu->features & ARM_SMMU_FEAT_PRI)
1842 arm_smmu_priq_thread(irq, dev);
1847 static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
1849 arm_smmu_gerror_handler(irq, dev);
1850 return IRQ_WAKE_THREAD;
1854 arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
1855 struct arm_smmu_cmdq_ent *cmd)
1859 /* ATC invalidates are always on 4096-bytes pages */
1860 size_t inval_grain_shift = 12;
1861 unsigned long page_start, page_end;
1863 *cmd = (struct arm_smmu_cmdq_ent) {
1864 .opcode = CMDQ_OP_ATC_INV,
1865 .substream_valid = !!ssid,
1870 cmd->atc.size = ATC_INV_SIZE_ALL;
1874 page_start = iova >> inval_grain_shift;
1875 page_end = (iova + size - 1) >> inval_grain_shift;
1878 * In an ATS Invalidate Request, the address must be aligned on the
1879 * range size, which must be a power of two number of page sizes. We
1880 * thus have to choose between grossly over-invalidating the region, or
1881 * splitting the invalidation into multiple commands. For simplicity
1882 * we'll go with the first solution, but should refine it in the future
1883 * if multiple commands are shown to be more efficient.
1885 * Find the smallest power of two that covers the range. The most
1886 * significant differing bit between the start and end addresses,
1887 * fls(start ^ end), indicates the required span. For example:
1889 * We want to invalidate pages [8; 11]. This is already the ideal range:
1890 * x = 0b1000 ^ 0b1011 = 0b11
1891 * span = 1 << fls(x) = 4
1893 * To invalidate pages [7; 10], we need to invalidate [0; 15]:
1894 * x = 0b0111 ^ 0b1010 = 0b1101
1895 * span = 1 << fls(x) = 16
1897 log2_span = fls_long(page_start ^ page_end);
1898 span_mask = (1ULL << log2_span) - 1;
1900 page_start &= ~span_mask;
1902 cmd->atc.addr = page_start << inval_grain_shift;
1903 cmd->atc.size = log2_span;
1906 static int arm_smmu_atc_inv_master(struct arm_smmu_master *master,
1907 struct arm_smmu_cmdq_ent *cmd)
1911 if (!master->ats_enabled)
1914 for (i = 0; i < master->num_sids; i++) {
1915 cmd->atc.sid = master->sids[i];
1916 arm_smmu_cmdq_issue_cmd(master->smmu, cmd);
1919 return arm_smmu_cmdq_issue_sync(master->smmu);
1922 static int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
1923 int ssid, unsigned long iova, size_t size)
1926 unsigned long flags;
1927 struct arm_smmu_cmdq_ent cmd;
1928 struct arm_smmu_master *master;
1930 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
1934 * Ensure that we've completed prior invalidation of the main TLBs
1935 * before we read 'nr_ats_masters' in case of a concurrent call to
1936 * arm_smmu_enable_ats():
1938 * // unmap() // arm_smmu_enable_ats()
1939 * TLBI+SYNC atomic_inc(&nr_ats_masters);
1941 * atomic_read(&nr_ats_masters); pci_enable_ats() // writel()
1943 * Ensures that we always see the incremented 'nr_ats_masters' count if
1944 * ATS was enabled at the PCI device before completion of the TLBI.
1947 if (!atomic_read(&smmu_domain->nr_ats_masters))
1950 arm_smmu_atc_inv_to_cmd(ssid, iova, size, &cmd);
1952 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
1953 list_for_each_entry(master, &smmu_domain->devices, domain_head)
1954 ret |= arm_smmu_atc_inv_master(master, &cmd);
1955 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1957 return ret ? -ETIMEDOUT : 0;
1960 /* IO_PGTABLE API */
1961 static void arm_smmu_tlb_inv_context(void *cookie)
1963 struct arm_smmu_domain *smmu_domain = cookie;
1964 struct arm_smmu_device *smmu = smmu_domain->smmu;
1965 struct arm_smmu_cmdq_ent cmd;
1967 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1968 cmd.opcode = CMDQ_OP_TLBI_NH_ASID;
1969 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
1972 cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
1973 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
1977 * NOTE: when io-pgtable is in non-strict mode, we may get here with
1978 * PTEs previously cleared by unmaps on the current CPU not yet visible
1979 * to the SMMU. We are relying on the dma_wmb() implicit during cmd
1980 * insertion to guarantee those are observed before the TLBI. Do be
1983 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1984 arm_smmu_cmdq_issue_sync(smmu);
1985 arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);
1988 static void arm_smmu_tlb_inv_range(unsigned long iova, size_t size,
1989 size_t granule, bool leaf,
1990 struct arm_smmu_domain *smmu_domain)
1992 u64 cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS];
1993 struct arm_smmu_device *smmu = smmu_domain->smmu;
1994 unsigned long start = iova, end = iova + size;
1996 struct arm_smmu_cmdq_ent cmd = {
2005 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2006 cmd.opcode = CMDQ_OP_TLBI_NH_VA;
2007 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
2009 cmd.opcode = CMDQ_OP_TLBI_S2_IPA;
2010 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
2013 while (iova < end) {
2014 if (i == CMDQ_BATCH_ENTRIES) {
2015 arm_smmu_cmdq_issue_cmdlist(smmu, cmds, i, false);
2019 cmd.tlbi.addr = iova;
2020 arm_smmu_cmdq_build_cmd(&cmds[i * CMDQ_ENT_DWORDS], &cmd);
2025 arm_smmu_cmdq_issue_cmdlist(smmu, cmds, i, true);
2028 * Unfortunately, this can't be leaf-only since we may have
2029 * zapped an entire table.
2031 arm_smmu_atc_inv_domain(smmu_domain, 0, start, size);
2034 static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
2035 unsigned long iova, size_t granule,
2038 struct arm_smmu_domain *smmu_domain = cookie;
2039 struct iommu_domain *domain = &smmu_domain->domain;
2041 iommu_iotlb_gather_add_page(domain, gather, iova, granule);
2044 static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
2045 size_t granule, void *cookie)
2047 arm_smmu_tlb_inv_range(iova, size, granule, false, cookie);
2050 static void arm_smmu_tlb_inv_leaf(unsigned long iova, size_t size,
2051 size_t granule, void *cookie)
2053 arm_smmu_tlb_inv_range(iova, size, granule, true, cookie);
2056 static const struct iommu_flush_ops arm_smmu_flush_ops = {
2057 .tlb_flush_all = arm_smmu_tlb_inv_context,
2058 .tlb_flush_walk = arm_smmu_tlb_inv_walk,
2059 .tlb_flush_leaf = arm_smmu_tlb_inv_leaf,
2060 .tlb_add_page = arm_smmu_tlb_inv_page_nosync,
2064 static bool arm_smmu_capable(enum iommu_cap cap)
2067 case IOMMU_CAP_CACHE_COHERENCY:
2069 case IOMMU_CAP_NOEXEC:
2076 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
2078 struct arm_smmu_domain *smmu_domain;
2080 if (type != IOMMU_DOMAIN_UNMANAGED &&
2081 type != IOMMU_DOMAIN_DMA &&
2082 type != IOMMU_DOMAIN_IDENTITY)
2086 * Allocate the domain and initialise some of its data structures.
2087 * We can't really do anything meaningful until we've added a
2090 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
2094 if (type == IOMMU_DOMAIN_DMA &&
2095 iommu_get_dma_cookie(&smmu_domain->domain)) {
2100 mutex_init(&smmu_domain->init_mutex);
2101 INIT_LIST_HEAD(&smmu_domain->devices);
2102 spin_lock_init(&smmu_domain->devices_lock);
2104 return &smmu_domain->domain;
2107 static int arm_smmu_bitmap_alloc(unsigned long *map, int span)
2109 int idx, size = 1 << span;
2112 idx = find_first_zero_bit(map, size);
2115 } while (test_and_set_bit(idx, map));
2120 static void arm_smmu_bitmap_free(unsigned long *map, int idx)
2122 clear_bit(idx, map);
2125 static void arm_smmu_domain_free(struct iommu_domain *domain)
2127 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2128 struct arm_smmu_device *smmu = smmu_domain->smmu;
2130 iommu_put_dma_cookie(domain);
2131 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
2133 /* Free the CD and ASID, if we allocated them */
2134 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2135 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
2138 dmam_free_coherent(smmu_domain->smmu->dev,
2139 CTXDESC_CD_DWORDS << 3,
2143 arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid);
2146 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2148 arm_smmu_bitmap_free(smmu->vmid_map, cfg->vmid);
2154 static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
2155 struct io_pgtable_cfg *pgtbl_cfg)
2159 struct arm_smmu_device *smmu = smmu_domain->smmu;
2160 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
2162 asid = arm_smmu_bitmap_alloc(smmu->asid_map, smmu->asid_bits);
2166 cfg->cdptr = dmam_alloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3,
2168 GFP_KERNEL | __GFP_ZERO);
2170 dev_warn(smmu->dev, "failed to allocate context descriptor\n");
2175 cfg->cd.asid = (u16)asid;
2176 cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
2177 cfg->cd.tcr = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
2178 cfg->cd.mair = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
2182 arm_smmu_bitmap_free(smmu->asid_map, asid);
2186 static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,
2187 struct io_pgtable_cfg *pgtbl_cfg)
2190 struct arm_smmu_device *smmu = smmu_domain->smmu;
2191 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2193 vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits);
2197 cfg->vmid = (u16)vmid;
2198 cfg->vttbr = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
2199 cfg->vtcr = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
2203 static int arm_smmu_domain_finalise(struct iommu_domain *domain)
2206 unsigned long ias, oas;
2207 enum io_pgtable_fmt fmt;
2208 struct io_pgtable_cfg pgtbl_cfg;
2209 struct io_pgtable_ops *pgtbl_ops;
2210 int (*finalise_stage_fn)(struct arm_smmu_domain *,
2211 struct io_pgtable_cfg *);
2212 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2213 struct arm_smmu_device *smmu = smmu_domain->smmu;
2215 if (domain->type == IOMMU_DOMAIN_IDENTITY) {
2216 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
2220 /* Restrict the stage to what we can actually support */
2221 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
2222 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
2223 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
2224 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
2226 switch (smmu_domain->stage) {
2227 case ARM_SMMU_DOMAIN_S1:
2228 ias = (smmu->features & ARM_SMMU_FEAT_VAX) ? 52 : 48;
2229 ias = min_t(unsigned long, ias, VA_BITS);
2231 fmt = ARM_64_LPAE_S1;
2232 finalise_stage_fn = arm_smmu_domain_finalise_s1;
2234 case ARM_SMMU_DOMAIN_NESTED:
2235 case ARM_SMMU_DOMAIN_S2:
2238 fmt = ARM_64_LPAE_S2;
2239 finalise_stage_fn = arm_smmu_domain_finalise_s2;
2245 pgtbl_cfg = (struct io_pgtable_cfg) {
2246 .pgsize_bitmap = smmu->pgsize_bitmap,
2249 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENCY,
2250 .tlb = &arm_smmu_flush_ops,
2251 .iommu_dev = smmu->dev,
2254 if (smmu_domain->non_strict)
2255 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
2257 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
2261 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
2262 domain->geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;
2263 domain->geometry.force_aperture = true;
2265 ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg);
2267 free_io_pgtable_ops(pgtbl_ops);
2271 smmu_domain->pgtbl_ops = pgtbl_ops;
2275 static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
2278 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2280 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2281 struct arm_smmu_strtab_l1_desc *l1_desc;
2284 /* Two-level walk */
2285 idx = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS;
2286 l1_desc = &cfg->l1_desc[idx];
2287 idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
2288 step = &l1_desc->l2ptr[idx];
2290 /* Simple linear lookup */
2291 step = &cfg->strtab[sid * STRTAB_STE_DWORDS];
2297 static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master)
2300 struct arm_smmu_device *smmu = master->smmu;
2302 for (i = 0; i < master->num_sids; ++i) {
2303 u32 sid = master->sids[i];
2304 __le64 *step = arm_smmu_get_step_for_sid(smmu, sid);
2306 /* Bridged PCI devices may end up with duplicated IDs */
2307 for (j = 0; j < i; j++)
2308 if (master->sids[j] == sid)
2313 arm_smmu_write_strtab_ent(master, sid, step);
2317 #ifdef CONFIG_PCI_ATS
2318 static bool arm_smmu_ats_supported(struct arm_smmu_master *master)
2320 struct pci_dev *pdev;
2321 struct arm_smmu_device *smmu = master->smmu;
2322 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
2324 if (!(smmu->features & ARM_SMMU_FEAT_ATS) || !dev_is_pci(master->dev) ||
2325 !(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS) || pci_ats_disabled())
2328 pdev = to_pci_dev(master->dev);
2329 return !pdev->untrusted && pdev->ats_cap;
2332 static bool arm_smmu_ats_supported(struct arm_smmu_master *master)
2338 static void arm_smmu_enable_ats(struct arm_smmu_master *master)
2341 struct pci_dev *pdev;
2342 struct arm_smmu_device *smmu = master->smmu;
2343 struct arm_smmu_domain *smmu_domain = master->domain;
2345 /* Don't enable ATS at the endpoint if it's not enabled in the STE */
2346 if (!master->ats_enabled)
2349 /* Smallest Translation Unit: log2 of the smallest supported granule */
2350 stu = __ffs(smmu->pgsize_bitmap);
2351 pdev = to_pci_dev(master->dev);
2353 atomic_inc(&smmu_domain->nr_ats_masters);
2354 arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);
2355 if (pci_enable_ats(pdev, stu))
2356 dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu);
2359 static void arm_smmu_disable_ats(struct arm_smmu_master *master)
2361 struct arm_smmu_cmdq_ent cmd;
2362 struct arm_smmu_domain *smmu_domain = master->domain;
2364 if (!master->ats_enabled)
2367 pci_disable_ats(to_pci_dev(master->dev));
2369 * Ensure ATS is disabled at the endpoint before we issue the
2370 * ATC invalidation via the SMMU.
2373 arm_smmu_atc_inv_to_cmd(0, 0, 0, &cmd);
2374 arm_smmu_atc_inv_master(master, &cmd);
2375 atomic_dec(&smmu_domain->nr_ats_masters);
2378 static void arm_smmu_detach_dev(struct arm_smmu_master *master)
2380 unsigned long flags;
2381 struct arm_smmu_domain *smmu_domain = master->domain;
2386 arm_smmu_disable_ats(master);
2388 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2389 list_del(&master->domain_head);
2390 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2392 master->domain = NULL;
2393 master->ats_enabled = false;
2394 arm_smmu_install_ste_for_dev(master);
2397 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
2400 unsigned long flags;
2401 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2402 struct arm_smmu_device *smmu;
2403 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2404 struct arm_smmu_master *master;
2409 master = fwspec->iommu_priv;
2410 smmu = master->smmu;
2412 arm_smmu_detach_dev(master);
2414 mutex_lock(&smmu_domain->init_mutex);
2416 if (!smmu_domain->smmu) {
2417 smmu_domain->smmu = smmu;
2418 ret = arm_smmu_domain_finalise(domain);
2420 smmu_domain->smmu = NULL;
2423 } else if (smmu_domain->smmu != smmu) {
2425 "cannot attach to SMMU %s (upstream of %s)\n",
2426 dev_name(smmu_domain->smmu->dev),
2427 dev_name(smmu->dev));
2432 master->domain = smmu_domain;
2434 if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS)
2435 master->ats_enabled = arm_smmu_ats_supported(master);
2437 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
2438 arm_smmu_write_ctx_desc(smmu, &smmu_domain->s1_cfg);
2440 arm_smmu_install_ste_for_dev(master);
2442 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2443 list_add(&master->domain_head, &smmu_domain->devices);
2444 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2446 arm_smmu_enable_ats(master);
2449 mutex_unlock(&smmu_domain->init_mutex);
2453 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
2454 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
2456 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
2461 return ops->map(ops, iova, paddr, size, prot);
2464 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
2465 size_t size, struct iommu_iotlb_gather *gather)
2467 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2468 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
2473 return ops->unmap(ops, iova, size, gather);
2476 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
2478 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2480 if (smmu_domain->smmu)
2481 arm_smmu_tlb_inv_context(smmu_domain);
2484 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
2485 struct iommu_iotlb_gather *gather)
2487 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2489 arm_smmu_tlb_inv_range(gather->start, gather->end - gather->start,
2490 gather->pgsize, true, smmu_domain);
2494 arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
2496 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
2498 if (domain->type == IOMMU_DOMAIN_IDENTITY)
2504 return ops->iova_to_phys(ops, iova);
2507 static struct platform_driver arm_smmu_driver;
2510 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
2512 struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
2515 return dev ? dev_get_drvdata(dev) : NULL;
2518 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
2520 unsigned long limit = smmu->strtab_cfg.num_l1_ents;
2522 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2523 limit *= 1UL << STRTAB_SPLIT;
2528 static struct iommu_ops arm_smmu_ops;
2530 static int arm_smmu_add_device(struct device *dev)
2533 struct arm_smmu_device *smmu;
2534 struct arm_smmu_master *master;
2535 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2536 struct iommu_group *group;
2538 if (!fwspec || fwspec->ops != &arm_smmu_ops)
2541 * We _can_ actually withstand dodgy bus code re-calling add_device()
2542 * without an intervening remove_device()/of_xlate() sequence, but
2543 * we're not going to do so quietly...
2545 if (WARN_ON_ONCE(fwspec->iommu_priv)) {
2546 master = fwspec->iommu_priv;
2547 smmu = master->smmu;
2549 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
2552 master = kzalloc(sizeof(*master), GFP_KERNEL);
2557 master->smmu = smmu;
2558 master->sids = fwspec->ids;
2559 master->num_sids = fwspec->num_ids;
2560 fwspec->iommu_priv = master;
2563 /* Check the SIDs are in range of the SMMU and our stream table */
2564 for (i = 0; i < master->num_sids; i++) {
2565 u32 sid = master->sids[i];
2567 if (!arm_smmu_sid_in_range(smmu, sid))
2570 /* Ensure l2 strtab is initialised */
2571 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2572 ret = arm_smmu_init_l2_strtab(smmu, sid);
2578 group = iommu_group_get_for_dev(dev);
2579 if (!IS_ERR(group)) {
2580 iommu_group_put(group);
2581 iommu_device_link(&smmu->iommu, dev);
2584 return PTR_ERR_OR_ZERO(group);
2587 static void arm_smmu_remove_device(struct device *dev)
2589 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2590 struct arm_smmu_master *master;
2591 struct arm_smmu_device *smmu;
2593 if (!fwspec || fwspec->ops != &arm_smmu_ops)
2596 master = fwspec->iommu_priv;
2597 smmu = master->smmu;
2598 arm_smmu_detach_dev(master);
2599 iommu_group_remove_device(dev);
2600 iommu_device_unlink(&smmu->iommu, dev);
2602 iommu_fwspec_free(dev);
2605 static struct iommu_group *arm_smmu_device_group(struct device *dev)
2607 struct iommu_group *group;
2610 * We don't support devices sharing stream IDs other than PCI RID
2611 * aliases, since the necessary ID-to-device lookup becomes rather
2612 * impractical given a potential sparse 32-bit stream ID space.
2614 if (dev_is_pci(dev))
2615 group = pci_device_group(dev);
2617 group = generic_device_group(dev);
2622 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
2623 enum iommu_attr attr, void *data)
2625 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2627 switch (domain->type) {
2628 case IOMMU_DOMAIN_UNMANAGED:
2630 case DOMAIN_ATTR_NESTING:
2631 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
2637 case IOMMU_DOMAIN_DMA:
2639 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
2640 *(int *)data = smmu_domain->non_strict;
2651 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
2652 enum iommu_attr attr, void *data)
2655 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2657 mutex_lock(&smmu_domain->init_mutex);
2659 switch (domain->type) {
2660 case IOMMU_DOMAIN_UNMANAGED:
2662 case DOMAIN_ATTR_NESTING:
2663 if (smmu_domain->smmu) {
2669 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
2671 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
2677 case IOMMU_DOMAIN_DMA:
2679 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
2680 smmu_domain->non_strict = *(int *)data;
2691 mutex_unlock(&smmu_domain->init_mutex);
2695 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
2697 return iommu_fwspec_add_ids(dev, args->args, 1);
2700 static void arm_smmu_get_resv_regions(struct device *dev,
2701 struct list_head *head)
2703 struct iommu_resv_region *region;
2704 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
2706 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
2707 prot, IOMMU_RESV_SW_MSI);
2711 list_add_tail(®ion->list, head);
2713 iommu_dma_get_resv_regions(dev, head);
2716 static void arm_smmu_put_resv_regions(struct device *dev,
2717 struct list_head *head)
2719 struct iommu_resv_region *entry, *next;
2721 list_for_each_entry_safe(entry, next, head, list)
2725 static struct iommu_ops arm_smmu_ops = {
2726 .capable = arm_smmu_capable,
2727 .domain_alloc = arm_smmu_domain_alloc,
2728 .domain_free = arm_smmu_domain_free,
2729 .attach_dev = arm_smmu_attach_dev,
2730 .map = arm_smmu_map,
2731 .unmap = arm_smmu_unmap,
2732 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
2733 .iotlb_sync = arm_smmu_iotlb_sync,
2734 .iova_to_phys = arm_smmu_iova_to_phys,
2735 .add_device = arm_smmu_add_device,
2736 .remove_device = arm_smmu_remove_device,
2737 .device_group = arm_smmu_device_group,
2738 .domain_get_attr = arm_smmu_domain_get_attr,
2739 .domain_set_attr = arm_smmu_domain_set_attr,
2740 .of_xlate = arm_smmu_of_xlate,
2741 .get_resv_regions = arm_smmu_get_resv_regions,
2742 .put_resv_regions = arm_smmu_put_resv_regions,
2743 .pgsize_bitmap = -1UL, /* Restricted during device attach */
2746 /* Probing and initialisation functions */
2747 static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
2748 struct arm_smmu_queue *q,
2749 unsigned long prod_off,
2750 unsigned long cons_off,
2751 size_t dwords, const char *name)
2756 qsz = ((1 << q->llq.max_n_shift) * dwords) << 3;
2757 q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma,
2759 if (q->base || qsz < PAGE_SIZE)
2762 q->llq.max_n_shift--;
2767 "failed to allocate queue (0x%zx bytes) for %s\n",
2772 if (!WARN_ON(q->base_dma & (qsz - 1))) {
2773 dev_info(smmu->dev, "allocated %u entries for %s\n",
2774 1 << q->llq.max_n_shift, name);
2777 q->prod_reg = arm_smmu_page1_fixup(prod_off, smmu);
2778 q->cons_reg = arm_smmu_page1_fixup(cons_off, smmu);
2779 q->ent_dwords = dwords;
2781 q->q_base = Q_BASE_RWA;
2782 q->q_base |= q->base_dma & Q_BASE_ADDR_MASK;
2783 q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->llq.max_n_shift);
2785 q->llq.prod = q->llq.cons = 0;
2789 static void arm_smmu_cmdq_free_bitmap(void *data)
2791 unsigned long *bitmap = data;
2792 bitmap_free(bitmap);
2795 static int arm_smmu_cmdq_init(struct arm_smmu_device *smmu)
2798 struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
2799 unsigned int nents = 1 << cmdq->q.llq.max_n_shift;
2800 atomic_long_t *bitmap;
2802 atomic_set(&cmdq->owner_prod, 0);
2803 atomic_set(&cmdq->lock, 0);
2805 bitmap = (atomic_long_t *)bitmap_zalloc(nents, GFP_KERNEL);
2807 dev_err(smmu->dev, "failed to allocate cmdq bitmap\n");
2810 cmdq->valid_map = bitmap;
2811 devm_add_action(smmu->dev, arm_smmu_cmdq_free_bitmap, bitmap);
2817 static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
2822 ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD,
2823 ARM_SMMU_CMDQ_CONS, CMDQ_ENT_DWORDS,
2828 ret = arm_smmu_cmdq_init(smmu);
2833 ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD,
2834 ARM_SMMU_EVTQ_CONS, EVTQ_ENT_DWORDS,
2840 if (!(smmu->features & ARM_SMMU_FEAT_PRI))
2843 return arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD,
2844 ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS,
2848 static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu)
2851 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2852 size_t size = sizeof(*cfg->l1_desc) * cfg->num_l1_ents;
2853 void *strtab = smmu->strtab_cfg.strtab;
2855 cfg->l1_desc = devm_kzalloc(smmu->dev, size, GFP_KERNEL);
2856 if (!cfg->l1_desc) {
2857 dev_err(smmu->dev, "failed to allocate l1 stream table desc\n");
2861 for (i = 0; i < cfg->num_l1_ents; ++i) {
2862 arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]);
2863 strtab += STRTAB_L1_DESC_DWORDS << 3;
2869 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
2874 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2876 /* Calculate the L1 size, capped to the SIDSIZE. */
2877 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
2878 size = min(size, smmu->sid_bits - STRTAB_SPLIT);
2879 cfg->num_l1_ents = 1 << size;
2881 size += STRTAB_SPLIT;
2882 if (size < smmu->sid_bits)
2884 "2-level strtab only covers %u/%u bits of SID\n",
2885 size, smmu->sid_bits);
2887 l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);
2888 strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
2889 GFP_KERNEL | __GFP_ZERO);
2892 "failed to allocate l1 stream table (%u bytes)\n",
2896 cfg->strtab = strtab;
2898 /* Configure strtab_base_cfg for 2 levels */
2899 reg = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_2LVL);
2900 reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, size);
2901 reg |= FIELD_PREP(STRTAB_BASE_CFG_SPLIT, STRTAB_SPLIT);
2902 cfg->strtab_base_cfg = reg;
2904 return arm_smmu_init_l1_strtab(smmu);
2907 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
2912 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2914 size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);
2915 strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma,
2916 GFP_KERNEL | __GFP_ZERO);
2919 "failed to allocate linear stream table (%u bytes)\n",
2923 cfg->strtab = strtab;
2924 cfg->num_l1_ents = 1 << smmu->sid_bits;
2926 /* Configure strtab_base_cfg for a linear table covering all SIDs */
2927 reg = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_LINEAR);
2928 reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits);
2929 cfg->strtab_base_cfg = reg;
2931 arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents);
2935 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
2940 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2941 ret = arm_smmu_init_strtab_2lvl(smmu);
2943 ret = arm_smmu_init_strtab_linear(smmu);
2948 /* Set the strtab base address */
2949 reg = smmu->strtab_cfg.strtab_dma & STRTAB_BASE_ADDR_MASK;
2950 reg |= STRTAB_BASE_RA;
2951 smmu->strtab_cfg.strtab_base = reg;
2953 /* Allocate the first VMID for stage-2 bypass STEs */
2954 set_bit(0, smmu->vmid_map);
2958 static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
2962 ret = arm_smmu_init_queues(smmu);
2966 return arm_smmu_init_strtab(smmu);
2969 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
2970 unsigned int reg_off, unsigned int ack_off)
2974 writel_relaxed(val, smmu->base + reg_off);
2975 return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
2976 1, ARM_SMMU_POLL_TIMEOUT_US);
2979 /* GBPA is "special" */
2980 static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)
2983 u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;
2985 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
2986 1, ARM_SMMU_POLL_TIMEOUT_US);
2992 writel_relaxed(reg | GBPA_UPDATE, gbpa);
2993 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
2994 1, ARM_SMMU_POLL_TIMEOUT_US);
2997 dev_err(smmu->dev, "GBPA not responding to update\n");
3001 static void arm_smmu_free_msis(void *data)
3003 struct device *dev = data;
3004 platform_msi_domain_free_irqs(dev);
3007 static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
3009 phys_addr_t doorbell;
3010 struct device *dev = msi_desc_to_dev(desc);
3011 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
3012 phys_addr_t *cfg = arm_smmu_msi_cfg[desc->platform.msi_index];
3014 doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;
3015 doorbell &= MSI_CFG0_ADDR_MASK;
3017 writeq_relaxed(doorbell, smmu->base + cfg[0]);
3018 writel_relaxed(msg->data, smmu->base + cfg[1]);
3019 writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);
3022 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu)
3024 struct msi_desc *desc;
3025 int ret, nvec = ARM_SMMU_MAX_MSIS;
3026 struct device *dev = smmu->dev;
3028 /* Clear the MSI address regs */
3029 writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
3030 writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
3032 if (smmu->features & ARM_SMMU_FEAT_PRI)
3033 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
3037 if (!(smmu->features & ARM_SMMU_FEAT_MSI))
3040 if (!dev->msi_domain) {
3041 dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n");
3045 /* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */
3046 ret = platform_msi_domain_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg);
3048 dev_warn(dev, "failed to allocate MSIs - falling back to wired irqs\n");
3052 for_each_msi_entry(desc, dev) {
3053 switch (desc->platform.msi_index) {
3054 case EVTQ_MSI_INDEX:
3055 smmu->evtq.q.irq = desc->irq;
3057 case GERROR_MSI_INDEX:
3058 smmu->gerr_irq = desc->irq;
3060 case PRIQ_MSI_INDEX:
3061 smmu->priq.q.irq = desc->irq;
3063 default: /* Unknown */
3068 /* Add callback to free MSIs on teardown */
3069 devm_add_action(dev, arm_smmu_free_msis, dev);
3072 static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu)
3076 arm_smmu_setup_msis(smmu);
3078 /* Request interrupt lines */
3079 irq = smmu->evtq.q.irq;
3081 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
3082 arm_smmu_evtq_thread,
3084 "arm-smmu-v3-evtq", smmu);
3086 dev_warn(smmu->dev, "failed to enable evtq irq\n");
3088 dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n");
3091 irq = smmu->gerr_irq;
3093 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
3094 0, "arm-smmu-v3-gerror", smmu);
3096 dev_warn(smmu->dev, "failed to enable gerror irq\n");
3098 dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n");
3101 if (smmu->features & ARM_SMMU_FEAT_PRI) {
3102 irq = smmu->priq.q.irq;
3104 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
3105 arm_smmu_priq_thread,
3111 "failed to enable priq irq\n");
3113 dev_warn(smmu->dev, "no priq irq - PRI will be broken\n");
3118 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
3121 u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
3123 /* Disable IRQs first */
3124 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
3125 ARM_SMMU_IRQ_CTRLACK);
3127 dev_err(smmu->dev, "failed to disable irqs\n");
3131 irq = smmu->combined_irq;
3134 * Cavium ThunderX2 implementation doesn't support unique irq
3135 * lines. Use a single irq line for all the SMMUv3 interrupts.
3137 ret = devm_request_threaded_irq(smmu->dev, irq,
3138 arm_smmu_combined_irq_handler,
3139 arm_smmu_combined_irq_thread,
3141 "arm-smmu-v3-combined-irq", smmu);
3143 dev_warn(smmu->dev, "failed to enable combined irq\n");
3145 arm_smmu_setup_unique_irqs(smmu);
3147 if (smmu->features & ARM_SMMU_FEAT_PRI)
3148 irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
3150 /* Enable interrupt generation on the SMMU */
3151 ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
3152 ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
3154 dev_warn(smmu->dev, "failed to enable irqs\n");
3159 static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
3163 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
3165 dev_err(smmu->dev, "failed to clear cr0\n");
3170 static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass)
3174 struct arm_smmu_cmdq_ent cmd;
3176 /* Clear CR0 and sync (disables SMMU and queue processing) */
3177 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
3178 if (reg & CR0_SMMUEN) {
3179 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
3180 WARN_ON(is_kdump_kernel() && !disable_bypass);
3181 arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0);
3184 ret = arm_smmu_device_disable(smmu);
3188 /* CR1 (table and queue memory attributes) */
3189 reg = FIELD_PREP(CR1_TABLE_SH, ARM_SMMU_SH_ISH) |
3190 FIELD_PREP(CR1_TABLE_OC, CR1_CACHE_WB) |
3191 FIELD_PREP(CR1_TABLE_IC, CR1_CACHE_WB) |
3192 FIELD_PREP(CR1_QUEUE_SH, ARM_SMMU_SH_ISH) |
3193 FIELD_PREP(CR1_QUEUE_OC, CR1_CACHE_WB) |
3194 FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB);
3195 writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
3197 /* CR2 (random crap) */
3198 reg = CR2_PTM | CR2_RECINVSID | CR2_E2H;
3199 writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
3202 writeq_relaxed(smmu->strtab_cfg.strtab_base,
3203 smmu->base + ARM_SMMU_STRTAB_BASE);
3204 writel_relaxed(smmu->strtab_cfg.strtab_base_cfg,
3205 smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
3208 writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
3209 writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
3210 writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
3212 enables = CR0_CMDQEN;
3213 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3216 dev_err(smmu->dev, "failed to enable command queue\n");
3220 /* Invalidate any cached configuration */
3221 cmd.opcode = CMDQ_OP_CFGI_ALL;
3222 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
3223 arm_smmu_cmdq_issue_sync(smmu);
3225 /* Invalidate any stale TLB entries */
3226 if (smmu->features & ARM_SMMU_FEAT_HYP) {
3227 cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
3228 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
3231 cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
3232 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
3233 arm_smmu_cmdq_issue_sync(smmu);
3236 writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
3237 writel_relaxed(smmu->evtq.q.llq.prod,
3238 arm_smmu_page1_fixup(ARM_SMMU_EVTQ_PROD, smmu));
3239 writel_relaxed(smmu->evtq.q.llq.cons,
3240 arm_smmu_page1_fixup(ARM_SMMU_EVTQ_CONS, smmu));
3242 enables |= CR0_EVTQEN;
3243 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3246 dev_err(smmu->dev, "failed to enable event queue\n");
3251 if (smmu->features & ARM_SMMU_FEAT_PRI) {
3252 writeq_relaxed(smmu->priq.q.q_base,
3253 smmu->base + ARM_SMMU_PRIQ_BASE);
3254 writel_relaxed(smmu->priq.q.llq.prod,
3255 arm_smmu_page1_fixup(ARM_SMMU_PRIQ_PROD, smmu));
3256 writel_relaxed(smmu->priq.q.llq.cons,
3257 arm_smmu_page1_fixup(ARM_SMMU_PRIQ_CONS, smmu));
3259 enables |= CR0_PRIQEN;
3260 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3263 dev_err(smmu->dev, "failed to enable PRI queue\n");
3268 if (smmu->features & ARM_SMMU_FEAT_ATS) {
3269 enables |= CR0_ATSCHK;
3270 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3273 dev_err(smmu->dev, "failed to enable ATS check\n");
3278 ret = arm_smmu_setup_irqs(smmu);
3280 dev_err(smmu->dev, "failed to setup irqs\n");
3284 if (is_kdump_kernel())
3285 enables &= ~(CR0_EVTQEN | CR0_PRIQEN);
3287 /* Enable the SMMU interface, or ensure bypass */
3288 if (!bypass || disable_bypass) {
3289 enables |= CR0_SMMUEN;
3291 ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT);
3295 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3298 dev_err(smmu->dev, "failed to enable SMMU interface\n");
3305 static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
3308 bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY;
3311 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
3313 /* 2-level structures */
3314 if (FIELD_GET(IDR0_ST_LVL, reg) == IDR0_ST_LVL_2LVL)
3315 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
3317 if (reg & IDR0_CD2L)
3318 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
3321 * Translation table endianness.
3322 * We currently require the same endianness as the CPU, but this
3323 * could be changed later by adding a new IO_PGTABLE_QUIRK.
3325 switch (FIELD_GET(IDR0_TTENDIAN, reg)) {
3326 case IDR0_TTENDIAN_MIXED:
3327 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
3330 case IDR0_TTENDIAN_BE:
3331 smmu->features |= ARM_SMMU_FEAT_TT_BE;
3334 case IDR0_TTENDIAN_LE:
3335 smmu->features |= ARM_SMMU_FEAT_TT_LE;
3339 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
3343 /* Boolean feature flags */
3344 if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
3345 smmu->features |= ARM_SMMU_FEAT_PRI;
3347 if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
3348 smmu->features |= ARM_SMMU_FEAT_ATS;
3351 smmu->features |= ARM_SMMU_FEAT_SEV;
3354 smmu->features |= ARM_SMMU_FEAT_MSI;
3357 smmu->features |= ARM_SMMU_FEAT_HYP;
3360 * The coherency feature as set by FW is used in preference to the ID
3361 * register, but warn on mismatch.
3363 if (!!(reg & IDR0_COHACC) != coherent)
3364 dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n",
3365 coherent ? "true" : "false");
3367 switch (FIELD_GET(IDR0_STALL_MODEL, reg)) {
3368 case IDR0_STALL_MODEL_FORCE:
3369 smmu->features |= ARM_SMMU_FEAT_STALL_FORCE;
3371 case IDR0_STALL_MODEL_STALL:
3372 smmu->features |= ARM_SMMU_FEAT_STALLS;
3376 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
3379 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
3381 if (!(reg & (IDR0_S1P | IDR0_S2P))) {
3382 dev_err(smmu->dev, "no translation support!\n");
3386 /* We only support the AArch64 table format at present */
3387 switch (FIELD_GET(IDR0_TTF, reg)) {
3388 case IDR0_TTF_AARCH32_64:
3391 case IDR0_TTF_AARCH64:
3394 dev_err(smmu->dev, "AArch64 table format not supported!\n");
3398 /* ASID/VMID sizes */
3399 smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
3400 smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
3403 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
3404 if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
3405 dev_err(smmu->dev, "embedded implementation not supported\n");
3409 /* Queue sizes, capped to ensure natural alignment */
3410 smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,
3411 FIELD_GET(IDR1_CMDQS, reg));
3412 if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) {
3414 * We don't support splitting up batches, so one batch of
3415 * commands plus an extra sync needs to fit inside the command
3416 * queue. There's also no way we can handle the weird alignment
3417 * restrictions on the base pointer for a unit-length queue.
3419 dev_err(smmu->dev, "command queue size <= %d entries not supported\n",
3420 CMDQ_BATCH_ENTRIES);
3424 smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT,
3425 FIELD_GET(IDR1_EVTQS, reg));
3426 smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT,
3427 FIELD_GET(IDR1_PRIQS, reg));
3429 /* SID/SSID sizes */
3430 smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);
3431 smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg);
3434 * If the SMMU supports fewer bits than would fill a single L2 stream
3435 * table, use a linear table instead.
3437 if (smmu->sid_bits <= STRTAB_SPLIT)
3438 smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;
3441 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
3443 /* Maximum number of outstanding stalls */
3444 smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg);
3447 if (reg & IDR5_GRAN64K)
3448 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
3449 if (reg & IDR5_GRAN16K)
3450 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
3451 if (reg & IDR5_GRAN4K)
3452 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
3454 /* Input address size */
3455 if (FIELD_GET(IDR5_VAX, reg) == IDR5_VAX_52_BIT)
3456 smmu->features |= ARM_SMMU_FEAT_VAX;
3458 /* Output address size */
3459 switch (FIELD_GET(IDR5_OAS, reg)) {
3460 case IDR5_OAS_32_BIT:
3463 case IDR5_OAS_36_BIT:
3466 case IDR5_OAS_40_BIT:
3469 case IDR5_OAS_42_BIT:
3472 case IDR5_OAS_44_BIT:
3475 case IDR5_OAS_52_BIT:
3477 smmu->pgsize_bitmap |= 1ULL << 42; /* 4TB */
3481 "unknown output address size. Truncating to 48-bit\n");
3483 case IDR5_OAS_48_BIT:
3487 if (arm_smmu_ops.pgsize_bitmap == -1UL)
3488 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
3490 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
3492 /* Set the DMA mask for our table walker */
3493 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
3495 "failed to set DMA mask for table walker\n");
3497 smmu->ias = max(smmu->ias, smmu->oas);
3499 dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
3500 smmu->ias, smmu->oas, smmu->features);
3505 static void acpi_smmu_get_options(u32 model, struct arm_smmu_device *smmu)
3508 case ACPI_IORT_SMMU_V3_CAVIUM_CN99XX:
3509 smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY;
3511 case ACPI_IORT_SMMU_V3_HISILICON_HI161X:
3512 smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH;
3516 dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options);
3519 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
3520 struct arm_smmu_device *smmu)
3522 struct acpi_iort_smmu_v3 *iort_smmu;
3523 struct device *dev = smmu->dev;
3524 struct acpi_iort_node *node;
3526 node = *(struct acpi_iort_node **)dev_get_platdata(dev);
3528 /* Retrieve SMMUv3 specific data */
3529 iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
3531 acpi_smmu_get_options(iort_smmu->model, smmu);
3533 if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
3534 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
3539 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
3540 struct arm_smmu_device *smmu)
3546 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
3547 struct arm_smmu_device *smmu)
3549 struct device *dev = &pdev->dev;
3553 if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))
3554 dev_err(dev, "missing #iommu-cells property\n");
3555 else if (cells != 1)
3556 dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);
3560 parse_driver_options(smmu);
3562 if (of_dma_is_coherent(dev->of_node))
3563 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
3568 static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu)
3570 if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY)
3576 static int arm_smmu_device_probe(struct platform_device *pdev)
3579 struct resource *res;
3580 resource_size_t ioaddr;
3581 struct arm_smmu_device *smmu;
3582 struct device *dev = &pdev->dev;
3585 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
3587 dev_err(dev, "failed to allocate arm_smmu_device\n");
3593 ret = arm_smmu_device_dt_probe(pdev, smmu);
3595 ret = arm_smmu_device_acpi_probe(pdev, smmu);
3600 /* Set bypass mode according to firmware probing result */
3604 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3605 if (resource_size(res) + 1 < arm_smmu_resource_size(smmu)) {
3606 dev_err(dev, "MMIO region too small (%pr)\n", res);
3609 ioaddr = res->start;
3611 smmu->base = devm_ioremap_resource(dev, res);
3612 if (IS_ERR(smmu->base))
3613 return PTR_ERR(smmu->base);
3615 /* Interrupt lines */
3617 irq = platform_get_irq_byname_optional(pdev, "combined");
3619 smmu->combined_irq = irq;
3621 irq = platform_get_irq_byname_optional(pdev, "eventq");
3623 smmu->evtq.q.irq = irq;
3625 irq = platform_get_irq_byname_optional(pdev, "priq");
3627 smmu->priq.q.irq = irq;
3629 irq = platform_get_irq_byname_optional(pdev, "gerror");
3631 smmu->gerr_irq = irq;
3634 ret = arm_smmu_device_hw_probe(smmu);
3638 /* Initialise in-memory data structures */
3639 ret = arm_smmu_init_structures(smmu);
3643 /* Record our private device structure */
3644 platform_set_drvdata(pdev, smmu);
3646 /* Reset the device */
3647 ret = arm_smmu_device_reset(smmu, bypass);
3651 /* And we're up. Go go go! */
3652 ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,
3653 "smmu3.%pa", &ioaddr);
3657 iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
3658 iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
3660 ret = iommu_device_register(&smmu->iommu);
3662 dev_err(dev, "Failed to register iommu\n");
3667 if (pci_bus_type.iommu_ops != &arm_smmu_ops) {
3669 ret = bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
3674 #ifdef CONFIG_ARM_AMBA
3675 if (amba_bustype.iommu_ops != &arm_smmu_ops) {
3676 ret = bus_set_iommu(&amba_bustype, &arm_smmu_ops);
3681 if (platform_bus_type.iommu_ops != &arm_smmu_ops) {
3682 ret = bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
3689 static void arm_smmu_device_shutdown(struct platform_device *pdev)
3691 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
3693 arm_smmu_device_disable(smmu);
3696 static const struct of_device_id arm_smmu_of_match[] = {
3697 { .compatible = "arm,smmu-v3", },
3701 static struct platform_driver arm_smmu_driver = {
3703 .name = "arm-smmu-v3",
3704 .of_match_table = of_match_ptr(arm_smmu_of_match),
3705 .suppress_bind_attrs = true,
3707 .probe = arm_smmu_device_probe,
3708 .shutdown = arm_smmu_device_shutdown,
3710 builtin_platform_driver(arm_smmu_driver);